2024-11-07 09:35:49,136 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-07 09:35:49,152 main DEBUG Took 0.013566 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-07 09:35:49,153 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-07 09:35:49,153 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-07 09:35:49,154 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-07 09:35:49,156 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 09:35:49,165 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-07 09:35:49,179 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 09:35:49,181 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 09:35:49,182 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 09:35:49,182 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 09:35:49,183 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 09:35:49,183 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 09:35:49,184 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 09:35:49,185 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 09:35:49,185 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 09:35:49,186 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 09:35:49,187 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 09:35:49,187 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 09:35:49,188 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 09:35:49,188 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 09:35:49,189 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 09:35:49,189 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 09:35:49,190 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 09:35:49,190 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 09:35:49,191 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 09:35:49,191 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 09:35:49,192 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 09:35:49,193 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 09:35:49,193 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 09:35:49,194 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 09:35:49,194 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 09:35:49,195 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-07 09:35:49,196 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 09:35:49,198 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-07 09:35:49,200 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-07 09:35:49,201 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-07 09:35:49,202 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-07 09:35:49,202 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-07 09:35:49,214 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-07 09:35:49,217 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-07 09:35:49,220 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-07 09:35:49,220 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-07 09:35:49,220 main DEBUG createAppenders(={Console}) 2024-11-07 09:35:49,221 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-07 09:35:49,221 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-07 09:35:49,222 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-07 09:35:49,222 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-07 09:35:49,223 main DEBUG OutputStream closed 2024-11-07 09:35:49,223 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-07 09:35:49,223 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-07 09:35:49,224 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-07 09:35:49,309 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-07 09:35:49,312 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-07 09:35:49,314 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-07 09:35:49,315 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-07 09:35:49,316 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-07 09:35:49,316 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-07 09:35:49,317 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-07 09:35:49,317 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-07 09:35:49,318 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-07 09:35:49,318 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-07 09:35:49,319 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-07 09:35:49,319 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-07 09:35:49,320 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-07 09:35:49,320 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-07 09:35:49,320 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-07 09:35:49,321 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-07 09:35:49,321 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-07 09:35:49,322 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-07 09:35:49,324 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-07 09:35:49,325 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-07 09:35:49,325 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-07 09:35:49,326 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-07T09:35:49,571 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5aa2b610-c3b5-4196-570b-200c5ef98b6f 2024-11-07 09:35:49,574 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-07 09:35:49,574 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-07T09:35:49,584 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-07T09:35:49,601 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-07T09:35:49,616 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5aa2b610-c3b5-4196-570b-200c5ef98b6f/cluster_ccb59956-e5ab-933e-47b6-10a1e2a3fb72, deleteOnExit=true 2024-11-07T09:35:49,616 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-07T09:35:49,617 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5aa2b610-c3b5-4196-570b-200c5ef98b6f/test.cache.data in system properties and HBase conf 2024-11-07T09:35:49,618 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5aa2b610-c3b5-4196-570b-200c5ef98b6f/hadoop.tmp.dir in system properties and HBase conf 2024-11-07T09:35:49,618 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5aa2b610-c3b5-4196-570b-200c5ef98b6f/hadoop.log.dir in system properties and HBase conf 2024-11-07T09:35:49,619 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5aa2b610-c3b5-4196-570b-200c5ef98b6f/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-07T09:35:49,619 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5aa2b610-c3b5-4196-570b-200c5ef98b6f/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-07T09:35:49,620 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-07T09:35:49,714 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-07T09:35:49,818 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T09:35:49,822 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5aa2b610-c3b5-4196-570b-200c5ef98b6f/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-07T09:35:49,823 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5aa2b610-c3b5-4196-570b-200c5ef98b6f/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-07T09:35:49,823 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5aa2b610-c3b5-4196-570b-200c5ef98b6f/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-07T09:35:49,824 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5aa2b610-c3b5-4196-570b-200c5ef98b6f/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-07T09:35:49,824 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5aa2b610-c3b5-4196-570b-200c5ef98b6f/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-07T09:35:49,825 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5aa2b610-c3b5-4196-570b-200c5ef98b6f/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-07T09:35:49,825 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5aa2b610-c3b5-4196-570b-200c5ef98b6f/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-07T09:35:49,826 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5aa2b610-c3b5-4196-570b-200c5ef98b6f/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-07T09:35:49,826 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5aa2b610-c3b5-4196-570b-200c5ef98b6f/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-07T09:35:49,826 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5aa2b610-c3b5-4196-570b-200c5ef98b6f/nfs.dump.dir in system properties and HBase conf 2024-11-07T09:35:49,827 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5aa2b610-c3b5-4196-570b-200c5ef98b6f/java.io.tmpdir in system properties and HBase conf 2024-11-07T09:35:49,827 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5aa2b610-c3b5-4196-570b-200c5ef98b6f/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-07T09:35:49,828 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5aa2b610-c3b5-4196-570b-200c5ef98b6f/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-07T09:35:49,828 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5aa2b610-c3b5-4196-570b-200c5ef98b6f/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-07T09:35:50,724 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-07T09:35:50,806 INFO [Time-limited test {}] log.Log(170): Logging initialized @2475ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-07T09:35:50,884 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-07T09:35:50,945 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-07T09:35:50,965 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-07T09:35:50,965 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-07T09:35:50,966 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-07T09:35:50,980 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-07T09:35:50,982 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1a58fd8a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5aa2b610-c3b5-4196-570b-200c5ef98b6f/hadoop.log.dir/,AVAILABLE} 2024-11-07T09:35:50,983 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@130d168e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-07T09:35:51,174 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2341a969{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5aa2b610-c3b5-4196-570b-200c5ef98b6f/java.io.tmpdir/jetty-localhost-33307-hadoop-hdfs-3_4_1-tests_jar-_-any-6704077679594160274/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-07T09:35:51,187 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2d064751{HTTP/1.1, (http/1.1)}{localhost:33307} 2024-11-07T09:35:51,187 INFO [Time-limited test {}] server.Server(415): Started @2856ms 2024-11-07T09:35:51,577 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-07T09:35:51,583 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-07T09:35:51,584 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-07T09:35:51,584 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-07T09:35:51,585 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-07T09:35:51,585 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@54a7b62e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5aa2b610-c3b5-4196-570b-200c5ef98b6f/hadoop.log.dir/,AVAILABLE} 2024-11-07T09:35:51,586 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@72782cb0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-07T09:35:51,709 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@292d50c4{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5aa2b610-c3b5-4196-570b-200c5ef98b6f/java.io.tmpdir/jetty-localhost-43053-hadoop-hdfs-3_4_1-tests_jar-_-any-13445880004106458327/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-07T09:35:51,710 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@667c6eba{HTTP/1.1, (http/1.1)}{localhost:43053} 2024-11-07T09:35:51,710 INFO [Time-limited test {}] server.Server(415): Started @3380ms 2024-11-07T09:35:51,766 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-07T09:35:52,265 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5aa2b610-c3b5-4196-570b-200c5ef98b6f/cluster_ccb59956-e5ab-933e-47b6-10a1e2a3fb72/data/data2/current/BP-1158119767-172.17.0.2-1730972150483/current, will proceed with Du for space computation calculation, 2024-11-07T09:35:52,265 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5aa2b610-c3b5-4196-570b-200c5ef98b6f/cluster_ccb59956-e5ab-933e-47b6-10a1e2a3fb72/data/data1/current/BP-1158119767-172.17.0.2-1730972150483/current, will proceed with Du for space computation calculation, 2024-11-07T09:35:52,301 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-07T09:35:52,351 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x156344895040970a with lease ID 0x4c96af2706a69ef4: Processing first storage report for DS-091f23f3-a7af-4630-9c90-a43dd5eedb51 from datanode DatanodeRegistration(127.0.0.1:34577, datanodeUuid=00b1f61d-c7c3-450c-b125-faf9dddf3851, infoPort=44059, infoSecurePort=0, ipcPort=40425, storageInfo=lv=-57;cid=testClusterID;nsid=1404123372;c=1730972150483) 2024-11-07T09:35:52,352 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x156344895040970a with lease ID 0x4c96af2706a69ef4: from storage DS-091f23f3-a7af-4630-9c90-a43dd5eedb51 node DatanodeRegistration(127.0.0.1:34577, datanodeUuid=00b1f61d-c7c3-450c-b125-faf9dddf3851, infoPort=44059, infoSecurePort=0, ipcPort=40425, storageInfo=lv=-57;cid=testClusterID;nsid=1404123372;c=1730972150483), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-07T09:35:52,352 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x156344895040970a with lease ID 0x4c96af2706a69ef4: Processing first storage report for DS-8a02f234-b7b1-4af2-8848-69f6c59324e2 from datanode DatanodeRegistration(127.0.0.1:34577, datanodeUuid=00b1f61d-c7c3-450c-b125-faf9dddf3851, infoPort=44059, infoSecurePort=0, ipcPort=40425, storageInfo=lv=-57;cid=testClusterID;nsid=1404123372;c=1730972150483) 2024-11-07T09:35:52,352 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x156344895040970a with lease ID 0x4c96af2706a69ef4: from storage DS-8a02f234-b7b1-4af2-8848-69f6c59324e2 node DatanodeRegistration(127.0.0.1:34577, datanodeUuid=00b1f61d-c7c3-450c-b125-faf9dddf3851, infoPort=44059, infoSecurePort=0, ipcPort=40425, storageInfo=lv=-57;cid=testClusterID;nsid=1404123372;c=1730972150483), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-07T09:35:52,445 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5aa2b610-c3b5-4196-570b-200c5ef98b6f 2024-11-07T09:35:52,518 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5aa2b610-c3b5-4196-570b-200c5ef98b6f/cluster_ccb59956-e5ab-933e-47b6-10a1e2a3fb72/zookeeper_0, clientPort=59189, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5aa2b610-c3b5-4196-570b-200c5ef98b6f/cluster_ccb59956-e5ab-933e-47b6-10a1e2a3fb72/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5aa2b610-c3b5-4196-570b-200c5ef98b6f/cluster_ccb59956-e5ab-933e-47b6-10a1e2a3fb72/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-07T09:35:52,529 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59189 2024-11-07T09:35:52,540 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T09:35:52,542 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T09:35:52,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741825_1001 (size=7) 2024-11-07T09:35:53,161 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260 with version=8 2024-11-07T09:35:53,162 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/hbase-staging 2024-11-07T09:35:53,258 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-07T09:35:53,506 INFO [Time-limited test {}] client.ConnectionUtils(128): master/9f1eb259a29b:0 server-side Connection retries=45 2024-11-07T09:35:53,517 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T09:35:53,517 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-07T09:35:53,522 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-07T09:35:53,522 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T09:35:53,522 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-07T09:35:53,658 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-07T09:35:53,717 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-07T09:35:53,725 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-07T09:35:53,729 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-07T09:35:53,756 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 8664 (auto-detected) 2024-11-07T09:35:53,757 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-07T09:35:53,775 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41689 2024-11-07T09:35:53,797 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41689 connecting to ZooKeeper ensemble=127.0.0.1:59189 2024-11-07T09:35:53,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:416890x0, quorum=127.0.0.1:59189, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-07T09:35:53,831 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41689-0x100f1c8dd440000 connected 2024-11-07T09:35:53,856 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T09:35:53,859 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T09:35:53,867 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41689-0x100f1c8dd440000, quorum=127.0.0.1:59189, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T09:35:53,872 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260, hbase.cluster.distributed=false 2024-11-07T09:35:53,893 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41689-0x100f1c8dd440000, quorum=127.0.0.1:59189, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-07T09:35:53,897 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41689 2024-11-07T09:35:53,897 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41689 2024-11-07T09:35:53,898 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41689 2024-11-07T09:35:53,898 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41689 2024-11-07T09:35:53,899 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41689 2024-11-07T09:35:54,010 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9f1eb259a29b:0 server-side Connection retries=45 2024-11-07T09:35:54,012 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T09:35:54,012 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-07T09:35:54,012 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-07T09:35:54,013 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T09:35:54,013 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-07T09:35:54,016 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-07T09:35:54,018 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-07T09:35:54,019 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42895 2024-11-07T09:35:54,021 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42895 connecting to ZooKeeper ensemble=127.0.0.1:59189 2024-11-07T09:35:54,022 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T09:35:54,025 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T09:35:54,031 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:428950x0, quorum=127.0.0.1:59189, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-07T09:35:54,032 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:428950x0, quorum=127.0.0.1:59189, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T09:35:54,032 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42895-0x100f1c8dd440001 connected 2024-11-07T09:35:54,036 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-07T09:35:54,044 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-07T09:35:54,046 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42895-0x100f1c8dd440001, quorum=127.0.0.1:59189, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-07T09:35:54,051 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42895-0x100f1c8dd440001, quorum=127.0.0.1:59189, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-07T09:35:54,052 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42895 2024-11-07T09:35:54,053 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42895 2024-11-07T09:35:54,053 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42895 2024-11-07T09:35:54,057 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42895 2024-11-07T09:35:54,057 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42895 2024-11-07T09:35:54,072 DEBUG [M:0;9f1eb259a29b:41689 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;9f1eb259a29b:41689 2024-11-07T09:35:54,074 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/9f1eb259a29b,41689,1730972153310 2024-11-07T09:35:54,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42895-0x100f1c8dd440001, quorum=127.0.0.1:59189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T09:35:54,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41689-0x100f1c8dd440000, quorum=127.0.0.1:59189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T09:35:54,082 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41689-0x100f1c8dd440000, quorum=127.0.0.1:59189, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/9f1eb259a29b,41689,1730972153310 2024-11-07T09:35:54,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42895-0x100f1c8dd440001, quorum=127.0.0.1:59189, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-07T09:35:54,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41689-0x100f1c8dd440000, quorum=127.0.0.1:59189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T09:35:54,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42895-0x100f1c8dd440001, quorum=127.0.0.1:59189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T09:35:54,104 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41689-0x100f1c8dd440000, quorum=127.0.0.1:59189, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-07T09:35:54,105 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/9f1eb259a29b,41689,1730972153310 from backup master directory 2024-11-07T09:35:54,109 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41689-0x100f1c8dd440000, quorum=127.0.0.1:59189, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/9f1eb259a29b,41689,1730972153310 2024-11-07T09:35:54,109 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42895-0x100f1c8dd440001, quorum=127.0.0.1:59189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T09:35:54,109 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41689-0x100f1c8dd440000, quorum=127.0.0.1:59189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T09:35:54,110 WARN [master/9f1eb259a29b:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-07T09:35:54,110 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=9f1eb259a29b,41689,1730972153310 2024-11-07T09:35:54,112 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-07T09:35:54,113 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-07T09:35:54,169 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/hbase.id] with ID: 039cc6dc-c845-4b6b-b77c-d5657b196eb1 2024-11-07T09:35:54,169 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/.tmp/hbase.id 2024-11-07T09:35:54,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741826_1002 (size=42) 2024-11-07T09:35:54,581 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/.tmp/hbase.id]:[hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/hbase.id] 2024-11-07T09:35:54,625 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T09:35:54,631 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-07T09:35:54,651 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-07T09:35:54,654 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42895-0x100f1c8dd440001, quorum=127.0.0.1:59189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T09:35:54,654 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41689-0x100f1c8dd440000, quorum=127.0.0.1:59189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T09:35:54,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741827_1003 (size=196) 2024-11-07T09:35:55,087 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T09:35:55,089 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-07T09:35:55,106 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:35:55,110 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T09:35:55,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741828_1004 (size=1189) 2024-11-07T09:35:55,559 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/MasterData/data/master/store 2024-11-07T09:35:55,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741829_1005 (size=34) 2024-11-07T09:35:55,983 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-07T09:35:55,986 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T09:35:55,988 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-07T09:35:55,988 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T09:35:55,988 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T09:35:55,989 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-07T09:35:55,990 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T09:35:55,990 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T09:35:55,991 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1730972155987Disabling compacts and flushes for region at 1730972155987Disabling writes for close at 1730972155989 (+2 ms)Writing region close event to WAL at 1730972155990 (+1 ms)Closed at 1730972155990 2024-11-07T09:35:55,993 WARN [master/9f1eb259a29b:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/MasterData/data/master/store/.initializing 2024-11-07T09:35:55,994 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/MasterData/WALs/9f1eb259a29b,41689,1730972153310 2024-11-07T09:35:56,002 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-07T09:35:56,016 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9f1eb259a29b%2C41689%2C1730972153310, suffix=, logDir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/MasterData/WALs/9f1eb259a29b,41689,1730972153310, archiveDir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/MasterData/oldWALs, maxLogs=10 2024-11-07T09:35:56,039 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/MasterData/WALs/9f1eb259a29b,41689,1730972153310/9f1eb259a29b%2C41689%2C1730972153310.1730972156021, exclude list is [], retry=0 2024-11-07T09:35:56,056 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34577,DS-091f23f3-a7af-4630-9c90-a43dd5eedb51,DISK] 2024-11-07T09:35:56,059 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-07T09:35:56,094 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/MasterData/WALs/9f1eb259a29b,41689,1730972153310/9f1eb259a29b%2C41689%2C1730972153310.1730972156021 2024-11-07T09:35:56,095 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44059:44059)] 2024-11-07T09:35:56,096 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-07T09:35:56,097 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T09:35:56,100 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T09:35:56,101 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T09:35:56,137 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T09:35:56,164 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-07T09:35:56,168 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:35:56,170 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T09:35:56,171 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T09:35:56,174 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-07T09:35:56,175 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:35:56,176 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T09:35:56,176 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T09:35:56,179 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-07T09:35:56,179 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:35:56,180 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T09:35:56,180 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T09:35:56,182 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-07T09:35:56,182 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:35:56,183 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T09:35:56,184 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T09:35:56,187 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-07T09:35:56,188 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-07T09:35:56,193 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T09:35:56,194 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T09:35:56,198 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-07T09:35:56,203 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T09:35:56,207 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T09:35:56,208 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63878567, jitterRate=-0.0481351763010025}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-07T09:35:56,213 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1730972156113Initializing all the Stores at 1730972156115 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730972156116 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730972156116Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730972156116Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730972156117 (+1 ms)Cleaning up temporary data from old regions at 1730972156194 (+77 ms)Region opened successfully at 1730972156213 (+19 ms) 2024-11-07T09:35:56,214 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-07T09:35:56,246 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27857d4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9f1eb259a29b/172.17.0.2:0 2024-11-07T09:35:56,277 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-07T09:35:56,288 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-07T09:35:56,288 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-07T09:35:56,291 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T09:35:56,292 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-07T09:35:56,297 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-07T09:35:56,297 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-07T09:35:56,322 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-07T09:35:56,330 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41689-0x100f1c8dd440000, quorum=127.0.0.1:59189, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-07T09:35:56,332 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-07T09:35:56,334 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-07T09:35:56,335 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41689-0x100f1c8dd440000, quorum=127.0.0.1:59189, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-07T09:35:56,337 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-07T09:35:56,339 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-07T09:35:56,343 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41689-0x100f1c8dd440000, quorum=127.0.0.1:59189, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-07T09:35:56,345 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-07T09:35:56,346 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41689-0x100f1c8dd440000, quorum=127.0.0.1:59189, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-07T09:35:56,348 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-07T09:35:56,365 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41689-0x100f1c8dd440000, quorum=127.0.0.1:59189, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-07T09:35:56,367 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-07T09:35:56,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42895-0x100f1c8dd440001, quorum=127.0.0.1:59189, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-07T09:35:56,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41689-0x100f1c8dd440000, quorum=127.0.0.1:59189, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-07T09:35:56,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42895-0x100f1c8dd440001, quorum=127.0.0.1:59189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T09:35:56,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41689-0x100f1c8dd440000, quorum=127.0.0.1:59189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T09:35:56,373 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=9f1eb259a29b,41689,1730972153310, sessionid=0x100f1c8dd440000, setting cluster-up flag (Was=false) 2024-11-07T09:35:56,386 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42895-0x100f1c8dd440001, quorum=127.0.0.1:59189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T09:35:56,386 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41689-0x100f1c8dd440000, quorum=127.0.0.1:59189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T09:35:56,393 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-07T09:35:56,395 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9f1eb259a29b,41689,1730972153310 2024-11-07T09:35:56,400 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41689-0x100f1c8dd440000, quorum=127.0.0.1:59189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T09:35:56,400 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42895-0x100f1c8dd440001, quorum=127.0.0.1:59189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T09:35:56,407 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-07T09:35:56,408 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9f1eb259a29b,41689,1730972153310 2024-11-07T09:35:56,414 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-07T09:35:56,462 INFO [RS:0;9f1eb259a29b:42895 {}] regionserver.HRegionServer(746): ClusterId : 039cc6dc-c845-4b6b-b77c-d5657b196eb1 2024-11-07T09:35:56,465 DEBUG [RS:0;9f1eb259a29b:42895 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-07T09:35:56,470 DEBUG [RS:0;9f1eb259a29b:42895 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-07T09:35:56,470 DEBUG [RS:0;9f1eb259a29b:42895 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-07T09:35:56,474 DEBUG [RS:0;9f1eb259a29b:42895 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-07T09:35:56,474 DEBUG [RS:0;9f1eb259a29b:42895 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1283c177, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9f1eb259a29b/172.17.0.2:0 2024-11-07T09:35:56,490 DEBUG [RS:0;9f1eb259a29b:42895 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;9f1eb259a29b:42895 2024-11-07T09:35:56,489 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T09:35:56,492 INFO [RS:0;9f1eb259a29b:42895 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-07T09:35:56,493 INFO [RS:0;9f1eb259a29b:42895 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-07T09:35:56,493 DEBUG [RS:0;9f1eb259a29b:42895 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-07T09:35:56,495 INFO [RS:0;9f1eb259a29b:42895 {}] regionserver.HRegionServer(2659): reportForDuty to master=9f1eb259a29b,41689,1730972153310 with port=42895, startcode=1730972153970 2024-11-07T09:35:56,500 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-07T09:35:56,506 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-07T09:35:56,507 DEBUG [RS:0;9f1eb259a29b:42895 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-07T09:35:56,511 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 9f1eb259a29b,41689,1730972153310 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-07T09:35:56,519 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/9f1eb259a29b:0, corePoolSize=5, maxPoolSize=5 2024-11-07T09:35:56,519 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/9f1eb259a29b:0, corePoolSize=5, maxPoolSize=5 2024-11-07T09:35:56,519 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/9f1eb259a29b:0, corePoolSize=5, maxPoolSize=5 2024-11-07T09:35:56,520 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/9f1eb259a29b:0, corePoolSize=5, maxPoolSize=5 2024-11-07T09:35:56,520 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/9f1eb259a29b:0, corePoolSize=10, maxPoolSize=10 2024-11-07T09:35:56,520 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/9f1eb259a29b:0, corePoolSize=1, maxPoolSize=1 2024-11-07T09:35:56,521 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/9f1eb259a29b:0, corePoolSize=2, maxPoolSize=2 2024-11-07T09:35:56,521 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/9f1eb259a29b:0, corePoolSize=1, maxPoolSize=1 2024-11-07T09:35:56,524 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1730972186524 2024-11-07T09:35:56,525 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-07T09:35:56,526 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-07T09:35:56,529 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-07T09:35:56,530 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-07T09:35:56,531 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-07T09:35:56,531 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-07T09:35:56,531 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-07T09:35:56,531 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-07T09:35:56,532 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-07T09:35:56,536 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-07T09:35:56,537 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-07T09:35:56,538 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:35:56,538 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-07T09:35:56,538 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-07T09:35:56,541 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-07T09:35:56,542 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-07T09:35:56,543 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44529, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-07T09:35:56,544 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/9f1eb259a29b:0:becomeActiveMaster-HFileCleaner.large.0-1730972156543,5,FailOnTimeoutGroup] 2024-11-07T09:35:56,544 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/9f1eb259a29b:0:becomeActiveMaster-HFileCleaner.small.0-1730972156544,5,FailOnTimeoutGroup] 2024-11-07T09:35:56,545 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-07T09:35:56,545 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-07T09:35:56,546 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-07T09:35:56,547 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-07T09:35:56,549 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41689 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9f1eb259a29b,42895,1730972153970 2024-11-07T09:35:56,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741831_1007 (size=1321) 2024-11-07T09:35:56,552 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41689 {}] master.ServerManager(517): Registering regionserver=9f1eb259a29b,42895,1730972153970 2024-11-07T09:35:56,566 DEBUG [RS:0;9f1eb259a29b:42895 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260 2024-11-07T09:35:56,566 DEBUG [RS:0;9f1eb259a29b:42895 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35443 2024-11-07T09:35:56,566 DEBUG [RS:0;9f1eb259a29b:42895 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-07T09:35:56,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41689-0x100f1c8dd440000, quorum=127.0.0.1:59189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-07T09:35:56,572 DEBUG [RS:0;9f1eb259a29b:42895 {}] zookeeper.ZKUtil(111): regionserver:42895-0x100f1c8dd440001, quorum=127.0.0.1:59189, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9f1eb259a29b,42895,1730972153970 2024-11-07T09:35:56,572 WARN [RS:0;9f1eb259a29b:42895 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-07T09:35:56,573 INFO [RS:0;9f1eb259a29b:42895 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T09:35:56,573 DEBUG [RS:0;9f1eb259a29b:42895 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/WALs/9f1eb259a29b,42895,1730972153970 2024-11-07T09:35:56,575 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9f1eb259a29b,42895,1730972153970] 2024-11-07T09:35:56,599 INFO [RS:0;9f1eb259a29b:42895 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-07T09:35:56,612 INFO [RS:0;9f1eb259a29b:42895 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-07T09:35:56,617 INFO [RS:0;9f1eb259a29b:42895 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-07T09:35:56,617 INFO [RS:0;9f1eb259a29b:42895 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T09:35:56,618 INFO [RS:0;9f1eb259a29b:42895 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-07T09:35:56,624 INFO [RS:0;9f1eb259a29b:42895 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-07T09:35:56,625 INFO [RS:0;9f1eb259a29b:42895 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-07T09:35:56,626 DEBUG [RS:0;9f1eb259a29b:42895 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9f1eb259a29b:0, corePoolSize=1, maxPoolSize=1 2024-11-07T09:35:56,626 DEBUG [RS:0;9f1eb259a29b:42895 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9f1eb259a29b:0, corePoolSize=1, maxPoolSize=1 2024-11-07T09:35:56,626 DEBUG [RS:0;9f1eb259a29b:42895 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9f1eb259a29b:0, corePoolSize=1, maxPoolSize=1 2024-11-07T09:35:56,626 DEBUG [RS:0;9f1eb259a29b:42895 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9f1eb259a29b:0, corePoolSize=1, maxPoolSize=1 2024-11-07T09:35:56,626 DEBUG [RS:0;9f1eb259a29b:42895 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9f1eb259a29b:0, corePoolSize=1, maxPoolSize=1 2024-11-07T09:35:56,626 DEBUG [RS:0;9f1eb259a29b:42895 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9f1eb259a29b:0, corePoolSize=2, maxPoolSize=2 2024-11-07T09:35:56,627 DEBUG [RS:0;9f1eb259a29b:42895 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9f1eb259a29b:0, corePoolSize=1, maxPoolSize=1 2024-11-07T09:35:56,627 DEBUG [RS:0;9f1eb259a29b:42895 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9f1eb259a29b:0, corePoolSize=1, maxPoolSize=1 2024-11-07T09:35:56,627 DEBUG [RS:0;9f1eb259a29b:42895 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9f1eb259a29b:0, corePoolSize=1, maxPoolSize=1 2024-11-07T09:35:56,627 DEBUG [RS:0;9f1eb259a29b:42895 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9f1eb259a29b:0, corePoolSize=1, maxPoolSize=1 2024-11-07T09:35:56,627 DEBUG [RS:0;9f1eb259a29b:42895 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9f1eb259a29b:0, corePoolSize=1, maxPoolSize=1 2024-11-07T09:35:56,627 DEBUG [RS:0;9f1eb259a29b:42895 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9f1eb259a29b:0, corePoolSize=1, maxPoolSize=1 2024-11-07T09:35:56,628 DEBUG [RS:0;9f1eb259a29b:42895 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9f1eb259a29b:0, corePoolSize=3, maxPoolSize=3 2024-11-07T09:35:56,628 DEBUG [RS:0;9f1eb259a29b:42895 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0, corePoolSize=3, maxPoolSize=3 2024-11-07T09:35:56,629 INFO [RS:0;9f1eb259a29b:42895 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T09:35:56,629 INFO [RS:0;9f1eb259a29b:42895 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T09:35:56,629 INFO [RS:0;9f1eb259a29b:42895 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T09:35:56,629 INFO [RS:0;9f1eb259a29b:42895 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-07T09:35:56,629 INFO [RS:0;9f1eb259a29b:42895 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-07T09:35:56,630 INFO [RS:0;9f1eb259a29b:42895 {}] hbase.ChoreService(168): Chore ScheduledChore name=9f1eb259a29b,42895,1730972153970-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-07T09:35:56,650 INFO [RS:0;9f1eb259a29b:42895 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-07T09:35:56,652 INFO [RS:0;9f1eb259a29b:42895 {}] hbase.ChoreService(168): Chore ScheduledChore name=9f1eb259a29b,42895,1730972153970-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T09:35:56,652 INFO [RS:0;9f1eb259a29b:42895 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T09:35:56,653 INFO [RS:0;9f1eb259a29b:42895 {}] regionserver.Replication(171): 9f1eb259a29b,42895,1730972153970 started 2024-11-07T09:35:56,671 INFO [RS:0;9f1eb259a29b:42895 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T09:35:56,671 INFO [RS:0;9f1eb259a29b:42895 {}] regionserver.HRegionServer(1482): Serving as 9f1eb259a29b,42895,1730972153970, RpcServer on 9f1eb259a29b/172.17.0.2:42895, sessionid=0x100f1c8dd440001 2024-11-07T09:35:56,672 DEBUG [RS:0;9f1eb259a29b:42895 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-07T09:35:56,672 DEBUG [RS:0;9f1eb259a29b:42895 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9f1eb259a29b,42895,1730972153970 2024-11-07T09:35:56,673 DEBUG [RS:0;9f1eb259a29b:42895 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9f1eb259a29b,42895,1730972153970' 2024-11-07T09:35:56,673 DEBUG [RS:0;9f1eb259a29b:42895 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-07T09:35:56,674 DEBUG [RS:0;9f1eb259a29b:42895 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-07T09:35:56,674 DEBUG [RS:0;9f1eb259a29b:42895 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-07T09:35:56,674 DEBUG [RS:0;9f1eb259a29b:42895 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-07T09:35:56,675 DEBUG [RS:0;9f1eb259a29b:42895 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9f1eb259a29b,42895,1730972153970 2024-11-07T09:35:56,675 DEBUG [RS:0;9f1eb259a29b:42895 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9f1eb259a29b,42895,1730972153970' 2024-11-07T09:35:56,675 DEBUG [RS:0;9f1eb259a29b:42895 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-07T09:35:56,675 DEBUG [RS:0;9f1eb259a29b:42895 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-07T09:35:56,676 DEBUG [RS:0;9f1eb259a29b:42895 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-07T09:35:56,676 INFO [RS:0;9f1eb259a29b:42895 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-07T09:35:56,676 INFO [RS:0;9f1eb259a29b:42895 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-07T09:35:56,782 INFO [RS:0;9f1eb259a29b:42895 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-07T09:35:56,785 INFO [RS:0;9f1eb259a29b:42895 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9f1eb259a29b%2C42895%2C1730972153970, suffix=, logDir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/WALs/9f1eb259a29b,42895,1730972153970, archiveDir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/oldWALs, maxLogs=32 2024-11-07T09:35:56,802 DEBUG [RS:0;9f1eb259a29b:42895 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/WALs/9f1eb259a29b,42895,1730972153970/9f1eb259a29b%2C42895%2C1730972153970.1730972156787, exclude list is [], retry=0 2024-11-07T09:35:56,806 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34577,DS-091f23f3-a7af-4630-9c90-a43dd5eedb51,DISK] 2024-11-07T09:35:56,810 INFO [RS:0;9f1eb259a29b:42895 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/WALs/9f1eb259a29b,42895,1730972153970/9f1eb259a29b%2C42895%2C1730972153970.1730972156787 2024-11-07T09:35:56,810 DEBUG [RS:0;9f1eb259a29b:42895 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44059:44059)] 2024-11-07T09:35:56,954 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-07T09:35:56,954 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260 2024-11-07T09:35:56,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741833_1009 (size=32) 2024-11-07T09:35:57,365 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T09:35:57,367 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-07T09:35:57,370 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-07T09:35:57,370 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:35:57,371 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T09:35:57,371 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-07T09:35:57,373 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-07T09:35:57,373 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:35:57,374 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T09:35:57,374 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-07T09:35:57,377 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-07T09:35:57,377 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:35:57,378 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T09:35:57,378 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-07T09:35:57,380 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-07T09:35:57,381 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:35:57,381 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T09:35:57,382 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-07T09:35:57,383 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/hbase/meta/1588230740 2024-11-07T09:35:57,383 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/hbase/meta/1588230740 2024-11-07T09:35:57,386 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-07T09:35:57,386 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-07T09:35:57,387 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T09:35:57,389 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-07T09:35:57,392 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T09:35:57,393 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67657632, jitterRate=0.00817728042602539}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T09:35:57,396 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1730972157365Initializing all the Stores at 1730972157367 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730972157367Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730972157367Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730972157367Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730972157367Cleaning up temporary data from old regions at 1730972157386 (+19 ms)Region opened successfully at 1730972157396 (+10 ms) 2024-11-07T09:35:57,396 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-07T09:35:57,396 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-07T09:35:57,396 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-07T09:35:57,396 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-07T09:35:57,396 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-07T09:35:57,397 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-07T09:35:57,398 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1730972157396Disabling compacts and flushes for region at 1730972157396Disabling writes for close at 1730972157396Writing region close event to WAL at 1730972157397 (+1 ms)Closed at 1730972157397 2024-11-07T09:35:57,401 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-07T09:35:57,402 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-07T09:35:57,408 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-07T09:35:57,417 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T09:35:57,420 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-07T09:35:57,573 DEBUG [9f1eb259a29b:41689 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-07T09:35:57,585 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=9f1eb259a29b,42895,1730972153970 2024-11-07T09:35:57,591 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9f1eb259a29b,42895,1730972153970, state=OPENING 2024-11-07T09:35:57,596 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-07T09:35:57,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42895-0x100f1c8dd440001, quorum=127.0.0.1:59189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T09:35:57,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41689-0x100f1c8dd440000, quorum=127.0.0.1:59189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T09:35:57,599 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T09:35:57,599 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T09:35:57,600 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T09:35:57,603 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=9f1eb259a29b,42895,1730972153970}] 2024-11-07T09:35:57,777 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-07T09:35:57,780 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49989, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-07T09:35:57,790 INFO [RS_OPEN_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-07T09:35:57,790 INFO [RS_OPEN_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T09:35:57,791 INFO [RS_OPEN_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-07T09:35:57,794 INFO [RS_OPEN_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9f1eb259a29b%2C42895%2C1730972153970.meta, suffix=.meta, logDir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/WALs/9f1eb259a29b,42895,1730972153970, archiveDir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/oldWALs, maxLogs=32 2024-11-07T09:35:57,811 DEBUG [RS_OPEN_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/WALs/9f1eb259a29b,42895,1730972153970/9f1eb259a29b%2C42895%2C1730972153970.meta.1730972157796.meta, exclude list is [], retry=0 2024-11-07T09:35:57,814 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34577,DS-091f23f3-a7af-4630-9c90-a43dd5eedb51,DISK] 2024-11-07T09:35:57,818 INFO [RS_OPEN_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/WALs/9f1eb259a29b,42895,1730972153970/9f1eb259a29b%2C42895%2C1730972153970.meta.1730972157796.meta 2024-11-07T09:35:57,818 DEBUG [RS_OPEN_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44059:44059)] 2024-11-07T09:35:57,818 DEBUG [RS_OPEN_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-07T09:35:57,820 DEBUG [RS_OPEN_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-07T09:35:57,823 DEBUG [RS_OPEN_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-07T09:35:57,828 INFO [RS_OPEN_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-07T09:35:57,832 DEBUG [RS_OPEN_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-07T09:35:57,833 DEBUG [RS_OPEN_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T09:35:57,833 DEBUG [RS_OPEN_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-07T09:35:57,833 DEBUG [RS_OPEN_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-07T09:35:57,836 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-07T09:35:57,838 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-07T09:35:57,838 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:35:57,839 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T09:35:57,839 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-07T09:35:57,840 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-07T09:35:57,840 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:35:57,841 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T09:35:57,841 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-07T09:35:57,842 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-07T09:35:57,842 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:35:57,843 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T09:35:57,843 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-07T09:35:57,844 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-07T09:35:57,844 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:35:57,845 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T09:35:57,845 DEBUG [RS_OPEN_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-07T09:35:57,847 DEBUG [RS_OPEN_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/hbase/meta/1588230740 2024-11-07T09:35:57,849 DEBUG [RS_OPEN_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/hbase/meta/1588230740 2024-11-07T09:35:57,851 DEBUG [RS_OPEN_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-07T09:35:57,852 DEBUG [RS_OPEN_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-07T09:35:57,853 DEBUG [RS_OPEN_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T09:35:57,855 DEBUG [RS_OPEN_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-07T09:35:57,857 INFO [RS_OPEN_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66931807, jitterRate=-0.0026383548974990845}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T09:35:57,857 DEBUG [RS_OPEN_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-07T09:35:57,858 DEBUG [RS_OPEN_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1730972157833Writing region info on filesystem at 1730972157834 (+1 ms)Initializing all the Stores at 1730972157836 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730972157836Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730972157836Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730972157836Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730972157836Cleaning up temporary data from old regions at 1730972157852 (+16 ms)Running coprocessor post-open hooks at 1730972157857 (+5 ms)Region opened successfully at 1730972157858 (+1 ms) 2024-11-07T09:35:57,865 INFO [RS_OPEN_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1730972157768 2024-11-07T09:35:57,877 DEBUG [RS_OPEN_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-07T09:35:57,877 INFO [RS_OPEN_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-07T09:35:57,879 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=9f1eb259a29b,42895,1730972153970 2024-11-07T09:35:57,881 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9f1eb259a29b,42895,1730972153970, state=OPEN 2024-11-07T09:35:57,887 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42895-0x100f1c8dd440001, quorum=127.0.0.1:59189, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T09:35:57,887 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41689-0x100f1c8dd440000, quorum=127.0.0.1:59189, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T09:35:57,887 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T09:35:57,887 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T09:35:57,887 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:35:57,892 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-07T09:35:57,893 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9f1eb259a29b,42895,1730972153970 in 285 msec 2024-11-07T09:35:57,899 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-07T09:35:57,899 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 486 msec 2024-11-07T09:35:57,900 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-07T09:35:57,900 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-07T09:35:57,927 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:35:57,928 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:35:57,949 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:35:57,952 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51491, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:35:57,977 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.5330 sec 2024-11-07T09:35:57,978 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1730972157977, completionTime=-1 2024-11-07T09:35:57,980 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-07T09:35:57,980 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-07T09:35:58,004 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-07T09:35:58,004 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1730972218004 2024-11-07T09:35:58,004 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1730972278004 2024-11-07T09:35:58,004 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 23 msec 2024-11-07T09:35:58,006 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9f1eb259a29b,41689,1730972153310-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T09:35:58,006 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9f1eb259a29b,41689,1730972153310-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T09:35:58,006 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9f1eb259a29b,41689,1730972153310-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T09:35:58,008 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-9f1eb259a29b:41689, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T09:35:58,008 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-07T09:35:58,009 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-07T09:35:58,014 DEBUG [master/9f1eb259a29b:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-07T09:35:58,034 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.924sec 2024-11-07T09:35:58,035 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-07T09:35:58,036 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-07T09:35:58,037 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-07T09:35:58,038 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-07T09:35:58,038 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-07T09:35:58,039 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9f1eb259a29b,41689,1730972153310-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-07T09:35:58,039 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9f1eb259a29b,41689,1730972153310-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-07T09:35:58,047 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-07T09:35:58,048 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-07T09:35:58,048 INFO [master/9f1eb259a29b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9f1eb259a29b,41689,1730972153310-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T09:35:58,072 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c90ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:35:58,074 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-07T09:35:58,074 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-07T09:35:58,078 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:35:58,080 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:35:58,088 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:35:58,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:35:58,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:35:58,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a9eb085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:35:58,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:35:58,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:35:58,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:35:58,097 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42588, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:35:58,099 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ba7d24a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:35:58,100 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:35:58,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:35:58,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:35:58,109 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33418, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:35:58,129 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=9f1eb259a29b,41689,1730972153310 2024-11-07T09:35:58,141 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=315, ProcessCount=11, AvailableMemoryMB=5483 2024-11-07T09:35:58,150 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T09:35:58,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 9f1eb259a29b,41689,1730972153310 2024-11-07T09:35:58,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6e387abf 2024-11-07T09:35:58,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T09:35:58,183 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42590, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T09:35:58,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T09:35:58,190 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T09:35:58,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-07T09:35:58,207 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T09:35:58,209 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-07T09:35:58,210 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:35:58,212 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T09:35:58,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T09:35:58,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741835_1011 (size=959) 2024-11-07T09:35:58,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T09:35:58,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T09:35:58,629 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260 2024-11-07T09:35:58,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741836_1012 (size=53) 2024-11-07T09:35:58,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T09:35:59,038 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T09:35:59,038 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 116dccc5098610cd38ab507cb3daad26, disabling compactions & flushes 2024-11-07T09:35:59,039 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:35:59,039 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:35:59,039 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. after waiting 0 ms 2024-11-07T09:35:59,039 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:35:59,039 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:35:59,039 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 116dccc5098610cd38ab507cb3daad26: Waiting for close lock at 1730972159038Disabling compacts and flushes for region at 1730972159038Disabling writes for close at 1730972159039 (+1 ms)Writing region close event to WAL at 1730972159039Closed at 1730972159039 2024-11-07T09:35:59,041 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T09:35:59,047 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1730972159042"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730972159042"}]},"ts":"1730972159042"} 2024-11-07T09:35:59,051 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-07T09:35:59,053 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T09:35:59,055 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730972159053"}]},"ts":"1730972159053"} 2024-11-07T09:35:59,059 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-07T09:35:59,061 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=116dccc5098610cd38ab507cb3daad26, ASSIGN}] 2024-11-07T09:35:59,063 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=116dccc5098610cd38ab507cb3daad26, ASSIGN 2024-11-07T09:35:59,065 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=116dccc5098610cd38ab507cb3daad26, ASSIGN; state=OFFLINE, location=9f1eb259a29b,42895,1730972153970; forceNewPlan=false, retain=false 2024-11-07T09:35:59,217 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=116dccc5098610cd38ab507cb3daad26, regionState=OPENING, regionLocation=9f1eb259a29b,42895,1730972153970 2024-11-07T09:35:59,221 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=116dccc5098610cd38ab507cb3daad26, ASSIGN because future has completed 2024-11-07T09:35:59,222 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970}] 2024-11-07T09:35:59,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T09:35:59,382 INFO [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:35:59,382 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} 2024-11-07T09:35:59,383 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 116dccc5098610cd38ab507cb3daad26 2024-11-07T09:35:59,383 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T09:35:59,383 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 116dccc5098610cd38ab507cb3daad26 2024-11-07T09:35:59,383 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 116dccc5098610cd38ab507cb3daad26 2024-11-07T09:35:59,385 INFO [StoreOpener-116dccc5098610cd38ab507cb3daad26-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 116dccc5098610cd38ab507cb3daad26 2024-11-07T09:35:59,387 INFO [StoreOpener-116dccc5098610cd38ab507cb3daad26-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 116dccc5098610cd38ab507cb3daad26 columnFamilyName A 2024-11-07T09:35:59,388 DEBUG [StoreOpener-116dccc5098610cd38ab507cb3daad26-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:35:59,389 INFO [StoreOpener-116dccc5098610cd38ab507cb3daad26-1 {}] regionserver.HStore(327): Store=116dccc5098610cd38ab507cb3daad26/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T09:35:59,389 INFO [StoreOpener-116dccc5098610cd38ab507cb3daad26-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 116dccc5098610cd38ab507cb3daad26 2024-11-07T09:35:59,391 INFO [StoreOpener-116dccc5098610cd38ab507cb3daad26-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 116dccc5098610cd38ab507cb3daad26 columnFamilyName B 2024-11-07T09:35:59,391 DEBUG [StoreOpener-116dccc5098610cd38ab507cb3daad26-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:35:59,391 INFO [StoreOpener-116dccc5098610cd38ab507cb3daad26-1 {}] regionserver.HStore(327): Store=116dccc5098610cd38ab507cb3daad26/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T09:35:59,392 INFO [StoreOpener-116dccc5098610cd38ab507cb3daad26-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 116dccc5098610cd38ab507cb3daad26 2024-11-07T09:35:59,394 INFO [StoreOpener-116dccc5098610cd38ab507cb3daad26-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 116dccc5098610cd38ab507cb3daad26 columnFamilyName C 2024-11-07T09:35:59,394 DEBUG [StoreOpener-116dccc5098610cd38ab507cb3daad26-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:35:59,394 INFO [StoreOpener-116dccc5098610cd38ab507cb3daad26-1 {}] regionserver.HStore(327): Store=116dccc5098610cd38ab507cb3daad26/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T09:35:59,394 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 116dccc5098610cd38ab507cb3daad26 2024-11-07T09:35:59,396 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26 2024-11-07T09:35:59,396 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26 2024-11-07T09:35:59,398 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 116dccc5098610cd38ab507cb3daad26 2024-11-07T09:35:59,398 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 116dccc5098610cd38ab507cb3daad26 2024-11-07T09:35:59,399 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T09:35:59,401 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 116dccc5098610cd38ab507cb3daad26 2024-11-07T09:35:59,404 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T09:35:59,405 INFO [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 116dccc5098610cd38ab507cb3daad26; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68706596, jitterRate=0.023808062076568604}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T09:35:59,405 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 116dccc5098610cd38ab507cb3daad26 2024-11-07T09:35:59,406 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 116dccc5098610cd38ab507cb3daad26: Running coprocessor pre-open hook at 1730972159383Writing region info on filesystem at 1730972159383Initializing all the Stores at 1730972159385 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730972159385Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730972159385Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730972159385Cleaning up temporary data from old regions at 1730972159398 (+13 ms)Running coprocessor post-open hooks at 1730972159405 (+7 ms)Region opened successfully at 1730972159406 (+1 ms) 2024-11-07T09:35:59,408 INFO [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., pid=6, masterSystemTime=1730972159376 2024-11-07T09:35:59,411 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:35:59,411 INFO [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:35:59,412 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=116dccc5098610cd38ab507cb3daad26, regionState=OPEN, openSeqNum=2, regionLocation=9f1eb259a29b,42895,1730972153970 2024-11-07T09:35:59,416 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 because future has completed 2024-11-07T09:35:59,416 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41689 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=9f1eb259a29b,42895,1730972153970, table=TestAcidGuarantees, region=116dccc5098610cd38ab507cb3daad26. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-07T09:35:59,422 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-07T09:35:59,422 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 in 195 msec 2024-11-07T09:35:59,425 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-07T09:35:59,425 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=116dccc5098610cd38ab507cb3daad26, ASSIGN in 361 msec 2024-11-07T09:35:59,427 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T09:35:59,427 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730972159427"}]},"ts":"1730972159427"} 2024-11-07T09:35:59,430 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-07T09:35:59,431 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T09:35:59,434 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2370 sec 2024-11-07T09:36:00,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T09:36:00,364 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:36:00,372 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c8b07f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:00,372 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:36:00,372 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:36:00,374 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:36:00,374 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:36:00,374 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:36:00,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cb64462, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:00,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:36:00,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:36:00,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:00,377 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42608, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:36:00,378 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2766bc80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:00,379 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:36:00,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:36:00,381 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:36:00,383 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33430, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:36:00,385 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T09:36:00,387 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 9f1eb259a29b,41689,1730972153310 2024-11-07T09:36:00,387 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@685b7b00 2024-11-07T09:36:00,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T09:36:00,391 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42616, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T09:36:00,395 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ff18d51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:00,395 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:36:00,395 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:36:00,397 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:36:00,397 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:36:00,398 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:36:00,398 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@374defb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:00,398 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:36:00,399 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:36:00,400 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:00,402 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42622, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:36:00,403 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@678699a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:00,407 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@109bd06b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:00,407 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:36:00,407 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:36:00,409 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:36:00,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:36:00,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:36:00,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c85fb36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:00,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:36:00,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:36:00,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:00,413 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42648, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:36:00,414 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bee8057, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:00,415 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2af20e9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:00,415 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:36:00,416 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:36:00,417 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:36:00,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:36:00,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:36:00,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f953883, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:00,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:36:00,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:36:00,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:00,424 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42666, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:36:00,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32ce3c82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:00,426 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7387a89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:00,426 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:36:00,427 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:36:00,428 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:36:00,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:36:00,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:36:00,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2886558d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:00,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:36:00,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:36:00,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:00,432 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42690, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:36:00,433 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e35ee59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:00,434 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75a031a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:00,434 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:36:00,434 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:36:00,435 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:36:00,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:36:00,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:36:00,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61e7d372, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:00,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:36:00,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:36:00,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:00,438 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42706, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:36:00,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d5a2ba1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:00,440 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@276a94e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:00,440 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:36:00,440 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:36:00,441 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:36:00,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:36:00,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:36:00,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@565e7015, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:00,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:36:00,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:36:00,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:00,443 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42730, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:36:00,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ec8a10e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:00,445 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d403639, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:00,445 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:36:00,446 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:36:00,446 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:36:00,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:36:00,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:36:00,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8c75040, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:00,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:36:00,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:36:00,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:00,448 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42750, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:36:00,449 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7412929a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:00,453 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52d66d40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:00,453 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:36:00,453 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:36:00,454 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:36:00,454 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:36:00,455 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:36:00,455 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@686b7a0e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:00,455 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:36:00,455 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:36:00,455 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:00,456 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42770, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:36:00,457 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2de480bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:00,460 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7be0d766, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:00,461 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:36:00,461 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:36:00,462 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:36:00,462 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:36:00,462 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:36:00,462 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@492f9788, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:00,462 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:36:00,463 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:36:00,463 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:00,464 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42800, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:36:00,465 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@327b8f48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:00,469 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:36:00,469 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:36:00,469 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:36:00,472 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:36:00,472 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:36:00,472 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:36:00,472 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:36:00,473 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:36:00,473 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:36:00,474 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33442, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:36:00,475 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33450, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:36:00,475 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33434, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:36:00,477 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:36:00,479 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:36:00,479 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:36:00,481 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33466, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:36:00,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2] 2024-11-07T09:36:00,482 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2] 2024-11-07T09:36:00,482 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2] 2024-11-07T09:36:00,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2] 2024-11-07T09:36:00,490 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:36:00,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:36:00,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:36:00,496 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33482, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:36:00,500 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:36:00,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2] 2024-11-07T09:36:00,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:36:00,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:36:00,506 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33486, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:36:00,507 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:36:00,508 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:36:00,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:36:00,510 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:36:00,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:36:00,511 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:36:00,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:36:00,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:36:00,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:36:00,514 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33506, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:36:00,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2] 2024-11-07T09:36:00,516 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T09:36:00,517 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33512, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:36:00,518 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33492, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:36:00,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2] 2024-11-07T09:36:00,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2] 2024-11-07T09:36:00,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2] 2024-11-07T09:36:00,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-07T09:36:00,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-07T09:36:00,531 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T09:36:00,533 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T09:36:00,535 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T09:36:00,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 116dccc5098610cd38ab507cb3daad26 2024-11-07T09:36:00,574 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:36:00,578 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 116dccc5098610cd38ab507cb3daad26 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T09:36:00,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-07T09:36:00,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/e395db610c4e453cb184045ae4293e78 is 50, key is test_row_0/A:col10/1730972160575/Put/seqid=0 2024-11-07T09:36:00,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T09:36:00,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:00,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. as already flushing 2024-11-07T09:36:00,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:00,715 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:00,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:00,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:00,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741837_1013 (size=16681) 2024-11-07T09:36:00,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/e395db610c4e453cb184045ae4293e78 2024-11-07T09:36:00,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:00,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33506 deadline: 1730972220787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:00,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:00,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972220800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:00,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:00,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972220808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:00,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:00,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972220809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:00,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:00,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1730972220810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:00,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:00,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:00,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:00,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:00,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:00,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:00,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:00,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:00,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:00,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:00,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-07T09:36:00,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:00,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:00,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:00,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:00,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:00,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T09:36:00,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:00,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. as already flushing 2024-11-07T09:36:00,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:00,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/fe938fc26f89444f9ea7d7b631d666ab is 50, key is test_row_0/B:col10/1730972160575/Put/seqid=0 2024-11-07T09:36:00,908 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:00,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:00,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:00,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741838_1014 (size=12001) 2024-11-07T09:36:00,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/fe938fc26f89444f9ea7d7b631d666ab 2024-11-07T09:36:00,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:00,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1730972220955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:00,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:00,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:00,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:00,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:00,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972220956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:00,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:00,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972220955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:00,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:00,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:00,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:00,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:00,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:00,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:00,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972220956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:00,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:00,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:00,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:00,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:00,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:00,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33506 deadline: 1730972220956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:00,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:00,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:00,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:00,989 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/5b195c81a14742979ae62ae704dda98b is 50, key is test_row_0/C:col10/1730972160575/Put/seqid=0 2024-11-07T09:36:01,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741839_1015 (size=12001) 2024-11-07T09:36:01,033 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/5b195c81a14742979ae62ae704dda98b 2024-11-07T09:36:01,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/e395db610c4e453cb184045ae4293e78 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/e395db610c4e453cb184045ae4293e78 2024-11-07T09:36:01,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/e395db610c4e453cb184045ae4293e78, entries=250, sequenceid=13, filesize=16.3 K 2024-11-07T09:36:01,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/fe938fc26f89444f9ea7d7b631d666ab as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/fe938fc26f89444f9ea7d7b631d666ab 2024-11-07T09:36:01,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T09:36:01,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/fe938fc26f89444f9ea7d7b631d666ab, entries=150, sequenceid=13, filesize=11.7 K 2024-11-07T09:36:01,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:01,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. as already flushing 2024-11-07T09:36:01,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:01,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:01,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:01,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/5b195c81a14742979ae62ae704dda98b as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/5b195c81a14742979ae62ae704dda98b 2024-11-07T09:36:01,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:01,111 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/5b195c81a14742979ae62ae704dda98b, entries=150, sequenceid=13, filesize=11.7 K 2024-11-07T09:36:01,120 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 116dccc5098610cd38ab507cb3daad26 in 540ms, sequenceid=13, compaction requested=false 2024-11-07T09:36:01,131 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-07T09:36:01,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:01,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-07T09:36:01,181 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:36:01,182 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 116dccc5098610cd38ab507cb3daad26 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-07T09:36:01,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 116dccc5098610cd38ab507cb3daad26 2024-11-07T09:36:01,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/1927300554a54fe6abb33ee9a6a45d71 is 50, key is test_row_0/A:col10/1730972161166/Put/seqid=0 2024-11-07T09:36:01,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:01,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972221198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:01,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:01,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:01,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:01,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:01,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33506 deadline: 1730972221200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:01,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:01,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:01,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:01,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:01,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1730972221204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:01,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:01,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:01,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:01,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:01,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972221209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:01,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:01,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:01,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:01,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:01,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972221215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:01,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:01,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:01,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:01,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T09:36:01,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:01,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. as already flushing 2024-11-07T09:36:01,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:01,238 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:01,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:01,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:01,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741840_1016 (size=14341) 2024-11-07T09:36:01,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:01,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972221324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:01,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:01,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33506 deadline: 1730972221325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:01,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:01,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:01,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:01,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:01,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:01,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:01,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:01,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1730972221334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:01,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:01,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:01,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:01,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:01,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972221344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:01,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:01,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972221344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:01,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:01,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:01,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:01,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:01,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:01,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:01,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T09:36:01,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:01,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. as already flushing 2024-11-07T09:36:01,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:01,394 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:01,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:01,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:01,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T09:36:01,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:01,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. as already flushing 2024-11-07T09:36:01,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:01,555 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:01,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:01,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:01,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33506 deadline: 1730972221544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:01,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:01,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:01,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1730972221545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:01,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:01,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:01,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972221545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:01,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:01,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:01,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:01,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:01,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:01,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:01,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:01,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:01,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:01,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972221555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:01,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:01,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:01,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:01,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:01,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972221555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:01,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:01,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:01,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:01,654 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/1927300554a54fe6abb33ee9a6a45d71 2024-11-07T09:36:01,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-07T09:36:01,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/2ecd8476893642a193bf55c81cbb8f14 is 50, key is test_row_0/B:col10/1730972161166/Put/seqid=0 2024-11-07T09:36:01,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741841_1017 (size=12001) 2024-11-07T09:36:01,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/2ecd8476893642a193bf55c81cbb8f14 2024-11-07T09:36:01,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T09:36:01,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:01,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. as already flushing 2024-11-07T09:36:01,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:01,712 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:01,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:01,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:01,724 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/6e16b0752438417b8d1891ec389cfdf0 is 50, key is test_row_0/C:col10/1730972161166/Put/seqid=0 2024-11-07T09:36:01,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741842_1018 (size=12001) 2024-11-07T09:36:01,775 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/6e16b0752438417b8d1891ec389cfdf0 2024-11-07T09:36:01,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/1927300554a54fe6abb33ee9a6a45d71 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/1927300554a54fe6abb33ee9a6a45d71 2024-11-07T09:36:01,810 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/1927300554a54fe6abb33ee9a6a45d71, entries=200, sequenceid=39, filesize=14.0 K 2024-11-07T09:36:01,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/2ecd8476893642a193bf55c81cbb8f14 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/2ecd8476893642a193bf55c81cbb8f14 2024-11-07T09:36:01,842 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/2ecd8476893642a193bf55c81cbb8f14, entries=150, sequenceid=39, filesize=11.7 K 2024-11-07T09:36:01,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/6e16b0752438417b8d1891ec389cfdf0 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/6e16b0752438417b8d1891ec389cfdf0 2024-11-07T09:36:01,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/6e16b0752438417b8d1891ec389cfdf0, entries=150, sequenceid=39, filesize=11.7 K 2024-11-07T09:36:01,865 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 116dccc5098610cd38ab507cb3daad26 in 684ms, sequenceid=39, compaction requested=false 2024-11-07T09:36:01,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:01,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T09:36:01,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:01,869 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 116dccc5098610cd38ab507cb3daad26 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-07T09:36:01,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 116dccc5098610cd38ab507cb3daad26 2024-11-07T09:36:01,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. as already flushing 2024-11-07T09:36:01,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/8fbc2603ac0f4a5fb7ee6150720b56fd is 50, key is test_row_0/A:col10/1730972161195/Put/seqid=0 2024-11-07T09:36:01,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741843_1019 (size=14341) 2024-11-07T09:36:01,948 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/8fbc2603ac0f4a5fb7ee6150720b56fd 2024-11-07T09:36:01,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/e8abba69d23e411dad3b49880a0378c5 is 50, key is test_row_0/B:col10/1730972161195/Put/seqid=0 2024-11-07T09:36:01,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741844_1020 (size=12001) 2024-11-07T09:36:02,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:02,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33506 deadline: 1730972222025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:02,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:02,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:02,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:02,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:02,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972222050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:02,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:02,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:02,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:02,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:02,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1730972222052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:02,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:02,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972222053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:02,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:02,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:02,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:02,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:02,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:02,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:02,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:02,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972222053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:02,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:02,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:02,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:02,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:02,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33506 deadline: 1730972222163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:02,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:02,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:02,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:02,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:02,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1730972222175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:02,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:02,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:02,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:02,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:02,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972222175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:02,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:02,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:02,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:02,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:02,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972222176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:02,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:02,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:02,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:02,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:02,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972222176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:02,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:02,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:02,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:02,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:02,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33506 deadline: 1730972222374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:02,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:02,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:02,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:02,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:02,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1730972222386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:02,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:02,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:02,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972222387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:02,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:02,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:02,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:02,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:02,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:02,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:02,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972222387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:02,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:02,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:02,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:02,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:02,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972222394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:02,397 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/e8abba69d23e411dad3b49880a0378c5 2024-11-07T09:36:02,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:02,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:02,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:02,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/97f7de0297b14b82a283b221d4494165 is 50, key is test_row_0/C:col10/1730972161195/Put/seqid=0 2024-11-07T09:36:02,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741845_1021 (size=12001) 2024-11-07T09:36:02,449 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/97f7de0297b14b82a283b221d4494165 2024-11-07T09:36:02,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/8fbc2603ac0f4a5fb7ee6150720b56fd as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/8fbc2603ac0f4a5fb7ee6150720b56fd 2024-11-07T09:36:02,478 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/8fbc2603ac0f4a5fb7ee6150720b56fd, entries=200, sequenceid=50, filesize=14.0 K 2024-11-07T09:36:02,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/e8abba69d23e411dad3b49880a0378c5 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/e8abba69d23e411dad3b49880a0378c5 2024-11-07T09:36:02,499 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/e8abba69d23e411dad3b49880a0378c5, entries=150, sequenceid=50, filesize=11.7 K 2024-11-07T09:36:02,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/97f7de0297b14b82a283b221d4494165 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/97f7de0297b14b82a283b221d4494165 2024-11-07T09:36:02,516 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/97f7de0297b14b82a283b221d4494165, entries=150, sequenceid=50, filesize=11.7 K 2024-11-07T09:36:02,520 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 116dccc5098610cd38ab507cb3daad26 in 650ms, sequenceid=50, compaction requested=true 2024-11-07T09:36:02,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:02,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:02,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-07T09:36:02,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-07T09:36:02,527 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-07T09:36:02,527 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9880 sec 2024-11-07T09:36:02,533 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.0120 sec 2024-11-07T09:36:02,593 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-07T09:36:02,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-07T09:36:02,684 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:36:02,685 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T09:36:02,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-07T09:36:02,689 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T09:36:02,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 116dccc5098610cd38ab507cb3daad26 2024-11-07T09:36:02,692 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:36:02,692 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 116dccc5098610cd38ab507cb3daad26 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-07T09:36:02,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-07T09:36:02,696 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T09:36:02,696 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T09:36:02,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/ce0be2dcec52464295a84c717ebeeee8 is 50, key is test_row_0/A:col10/1730972162687/Put/seqid=0 2024-11-07T09:36:02,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:02,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:02,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972222708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:02,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33506 deadline: 1730972222707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:02,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:02,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:02,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:02,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:02,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:02,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:02,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:02,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1730972222711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:02,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:02,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:02,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:02,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:02,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972222714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:02,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741846_1022 (size=12001) 2024-11-07T09:36:02,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:02,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:02,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:02,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:02,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972222716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:02,724 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/ce0be2dcec52464295a84c717ebeeee8 2024-11-07T09:36:02,725 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:02,725 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:02,725 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:02,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/1105290ccee740d1b7a0017563b6ffa4 is 50, key is test_row_0/B:col10/1730972162687/Put/seqid=0 2024-11-07T09:36:02,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741847_1023 (size=12001) 2024-11-07T09:36:02,786 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/1105290ccee740d1b7a0017563b6ffa4 2024-11-07T09:36:02,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-07T09:36:02,812 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/39b6ac24adc7492bb4c77bce268e3405 is 50, key is test_row_0/C:col10/1730972162687/Put/seqid=0 2024-11-07T09:36:02,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741848_1024 (size=12001) 2024-11-07T09:36:02,825 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/39b6ac24adc7492bb4c77bce268e3405 2024-11-07T09:36:02,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:02,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972222824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:02,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:02,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:02,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:02,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:02,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33506 deadline: 1730972222824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:02,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:02,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:02,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1730972222825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:02,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:02,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:02,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:02,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:02,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:02,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:02,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972222833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:02,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:02,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:02,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:02,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:02,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972222833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:02,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:02,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:02,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:02,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/ce0be2dcec52464295a84c717ebeeee8 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/ce0be2dcec52464295a84c717ebeeee8 2024-11-07T09:36:02,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-07T09:36:02,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:02,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. as already flushing 2024-11-07T09:36:02,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:02,851 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:02,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:02,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:02,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/ce0be2dcec52464295a84c717ebeeee8, entries=150, sequenceid=76, filesize=11.7 K 2024-11-07T09:36:02,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/1105290ccee740d1b7a0017563b6ffa4 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/1105290ccee740d1b7a0017563b6ffa4 2024-11-07T09:36:02,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/1105290ccee740d1b7a0017563b6ffa4, entries=150, sequenceid=76, filesize=11.7 K 2024-11-07T09:36:02,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/39b6ac24adc7492bb4c77bce268e3405 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/39b6ac24adc7492bb4c77bce268e3405 2024-11-07T09:36:02,898 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/39b6ac24adc7492bb4c77bce268e3405, entries=150, sequenceid=76, filesize=11.7 K 2024-11-07T09:36:02,899 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 116dccc5098610cd38ab507cb3daad26 in 207ms, sequenceid=76, compaction requested=true 2024-11-07T09:36:02,899 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:02,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 116dccc5098610cd38ab507cb3daad26:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T09:36:02,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:02,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 116dccc5098610cd38ab507cb3daad26:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T09:36:02,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:02,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 116dccc5098610cd38ab507cb3daad26:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:36:02,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:36:02,904 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T09:36:02,904 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T09:36:02,909 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T09:36:02,911 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 116dccc5098610cd38ab507cb3daad26/B is initiating minor compaction (all files) 2024-11-07T09:36:02,911 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 116dccc5098610cd38ab507cb3daad26/B in TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:02,912 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/fe938fc26f89444f9ea7d7b631d666ab, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/2ecd8476893642a193bf55c81cbb8f14, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/e8abba69d23e411dad3b49880a0378c5, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/1105290ccee740d1b7a0017563b6ffa4] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp, totalSize=46.9 K 2024-11-07T09:36:02,913 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting fe938fc26f89444f9ea7d7b631d666ab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1730972160538 2024-11-07T09:36:02,913 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57364 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T09:36:02,914 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 116dccc5098610cd38ab507cb3daad26/A is initiating minor compaction (all files) 2024-11-07T09:36:02,914 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 116dccc5098610cd38ab507cb3daad26/A in TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:02,914 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/e395db610c4e453cb184045ae4293e78, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/1927300554a54fe6abb33ee9a6a45d71, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/8fbc2603ac0f4a5fb7ee6150720b56fd, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/ce0be2dcec52464295a84c717ebeeee8] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp, totalSize=56.0 K 2024-11-07T09:36:02,915 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ecd8476893642a193bf55c81cbb8f14, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1730972160792 2024-11-07T09:36:02,916 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting e395db610c4e453cb184045ae4293e78, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1730972160538 2024-11-07T09:36:02,916 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting e8abba69d23e411dad3b49880a0378c5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1730972161195 2024-11-07T09:36:02,916 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1927300554a54fe6abb33ee9a6a45d71, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1730972160792 2024-11-07T09:36:02,918 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8fbc2603ac0f4a5fb7ee6150720b56fd, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1730972161195 2024-11-07T09:36:02,918 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 1105290ccee740d1b7a0017563b6ffa4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1730972162012 2024-11-07T09:36:02,920 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce0be2dcec52464295a84c717ebeeee8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1730972162012 2024-11-07T09:36:02,968 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 116dccc5098610cd38ab507cb3daad26#B#compaction#12 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:02,969 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/76aa031fdcd34dd3949c9364ab4c5e49 is 50, key is test_row_0/B:col10/1730972162687/Put/seqid=0 2024-11-07T09:36:02,973 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 116dccc5098610cd38ab507cb3daad26#A#compaction#13 average throughput is 0.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:02,974 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/86ec91df06724153b224b65a4d52b582 is 50, key is test_row_0/A:col10/1730972162687/Put/seqid=0 2024-11-07T09:36:02,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741849_1025 (size=12139) 2024-11-07T09:36:02,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741850_1026 (size=12139) 2024-11-07T09:36:02,996 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/76aa031fdcd34dd3949c9364ab4c5e49 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/76aa031fdcd34dd3949c9364ab4c5e49 2024-11-07T09:36:03,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-07T09:36:03,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:03,007 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 116dccc5098610cd38ab507cb3daad26 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-07T09:36:03,008 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/86ec91df06724153b224b65a4d52b582 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/86ec91df06724153b224b65a4d52b582 2024-11-07T09:36:03,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-07T09:36:03,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/df505b5175df4652a5caa4fa80a960b8 is 50, key is test_row_0/A:col10/1730972162710/Put/seqid=0 2024-11-07T09:36:03,035 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 116dccc5098610cd38ab507cb3daad26/A of 116dccc5098610cd38ab507cb3daad26 into 86ec91df06724153b224b65a4d52b582(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:03,035 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:03,036 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 116dccc5098610cd38ab507cb3daad26/B of 116dccc5098610cd38ab507cb3daad26 into 76aa031fdcd34dd3949c9364ab4c5e49(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:03,036 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:03,039 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., storeName=116dccc5098610cd38ab507cb3daad26/A, priority=12, startTime=1730972162901; duration=0sec 2024-11-07T09:36:03,040 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., storeName=116dccc5098610cd38ab507cb3daad26/B, priority=12, startTime=1730972162904; duration=0sec 2024-11-07T09:36:03,040 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:36:03,041 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:36:03,041 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 116dccc5098610cd38ab507cb3daad26:A 2024-11-07T09:36:03,041 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 116dccc5098610cd38ab507cb3daad26:B 2024-11-07T09:36:03,041 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T09:36:03,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. as already flushing 2024-11-07T09:36:03,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 116dccc5098610cd38ab507cb3daad26 2024-11-07T09:36:03,051 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T09:36:03,052 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 116dccc5098610cd38ab507cb3daad26/C is initiating minor compaction (all files) 2024-11-07T09:36:03,052 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 116dccc5098610cd38ab507cb3daad26/C in TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:03,052 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/5b195c81a14742979ae62ae704dda98b, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/6e16b0752438417b8d1891ec389cfdf0, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/97f7de0297b14b82a283b221d4494165, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/39b6ac24adc7492bb4c77bce268e3405] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp, totalSize=46.9 K 2024-11-07T09:36:03,053 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b195c81a14742979ae62ae704dda98b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1730972160538 2024-11-07T09:36:03,054 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e16b0752438417b8d1891ec389cfdf0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1730972160792 2024-11-07T09:36:03,055 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97f7de0297b14b82a283b221d4494165, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1730972161195 2024-11-07T09:36:03,056 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39b6ac24adc7492bb4c77bce268e3405, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1730972162012 2024-11-07T09:36:03,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741851_1027 (size=12001) 2024-11-07T09:36:03,068 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/df505b5175df4652a5caa4fa80a960b8 2024-11-07T09:36:03,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/d417061d29314c99946b5975d4b7d332 is 50, key is test_row_0/B:col10/1730972162710/Put/seqid=0 2024-11-07T09:36:03,107 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 116dccc5098610cd38ab507cb3daad26#C#compaction#16 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:03,108 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/eb0801396c75482781c6dbfe4207462d is 50, key is test_row_0/C:col10/1730972162687/Put/seqid=0 2024-11-07T09:36:03,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:03,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972223100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:03,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:03,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:03,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:03,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:03,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972223105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:03,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:03,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1730972223106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:03,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:03,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:03,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:03,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:03,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33506 deadline: 1730972223108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:03,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:03,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:03,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:03,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:03,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:03,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:03,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741852_1028 (size=12001) 2024-11-07T09:36:03,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:03,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972223115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:03,123 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/d417061d29314c99946b5975d4b7d332 2024-11-07T09:36:03,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:03,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:03,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:03,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741853_1029 (size=12139) 2024-11-07T09:36:03,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/7f8174412ac14cf2b29f1bd83b5ffa6e is 50, key is test_row_0/C:col10/1730972162710/Put/seqid=0 2024-11-07T09:36:03,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741854_1030 (size=12001) 2024-11-07T09:36:03,178 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/7f8174412ac14cf2b29f1bd83b5ffa6e 2024-11-07T09:36:03,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/df505b5175df4652a5caa4fa80a960b8 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/df505b5175df4652a5caa4fa80a960b8 2024-11-07T09:36:03,205 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/df505b5175df4652a5caa4fa80a960b8, entries=150, sequenceid=86, filesize=11.7 K 2024-11-07T09:36:03,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/d417061d29314c99946b5975d4b7d332 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/d417061d29314c99946b5975d4b7d332 2024-11-07T09:36:03,218 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/d417061d29314c99946b5975d4b7d332, entries=150, sequenceid=86, filesize=11.7 K 2024-11-07T09:36:03,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/7f8174412ac14cf2b29f1bd83b5ffa6e as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/7f8174412ac14cf2b29f1bd83b5ffa6e 2024-11-07T09:36:03,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:03,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972223224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:03,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:03,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:03,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:03,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:03,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1730972223224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:03,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:03,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:03,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:03,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:03,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972223224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:03,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:03,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:03,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:03,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:03,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33506 deadline: 1730972223225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:03,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:03,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:03,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:03,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:03,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972223234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:03,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:03,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:03,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:03,240 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/7f8174412ac14cf2b29f1bd83b5ffa6e, entries=150, sequenceid=86, filesize=11.7 K 2024-11-07T09:36:03,244 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 116dccc5098610cd38ab507cb3daad26 in 237ms, sequenceid=86, compaction requested=false 2024-11-07T09:36:03,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:03,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:03,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-07T09:36:03,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-07T09:36:03,252 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-07T09:36:03,253 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 551 msec 2024-11-07T09:36:03,259 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 568 msec 2024-11-07T09:36:03,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-07T09:36:03,323 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:36:03,324 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T09:36:03,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-07T09:36:03,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-07T09:36:03,328 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T09:36:03,329 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T09:36:03,329 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T09:36:03,352 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T09:36:03,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-07T09:36:03,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 116dccc5098610cd38ab507cb3daad26 2024-11-07T09:36:03,437 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:36:03,438 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 116dccc5098610cd38ab507cb3daad26 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-07T09:36:03,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/5529f3fb6df5466283cb3183ee25ffa8 is 50, key is test_row_0/A:col10/1730972163097/Put/seqid=0 2024-11-07T09:36:03,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741855_1031 (size=14341) 2024-11-07T09:36:03,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-07T09:36:03,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:03,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. as already flushing 2024-11-07T09:36:03,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:03,484 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:03,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:03,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:03,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:03,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972223512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:03,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:03,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:03,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33506 deadline: 1730972223516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:03,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972223517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:03,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:03,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:03,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:03,530 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:03,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:03,531 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:03,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:03,531 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:03,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:03,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:03,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1730972223527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:03,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:03,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:03,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:03,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:03,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972223530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:03,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:03,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:03,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:03,563 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/eb0801396c75482781c6dbfe4207462d as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/eb0801396c75482781c6dbfe4207462d 2024-11-07T09:36:03,584 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 116dccc5098610cd38ab507cb3daad26/C of 116dccc5098610cd38ab507cb3daad26 into eb0801396c75482781c6dbfe4207462d(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:03,584 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:03,585 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., storeName=116dccc5098610cd38ab507cb3daad26/C, priority=12, startTime=1730972162904; duration=0sec 2024-11-07T09:36:03,585 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:03,585 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 116dccc5098610cd38ab507cb3daad26:C 2024-11-07T09:36:03,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:03,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972223633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:03,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:03,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:03,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:03,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:03,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33506 deadline: 1730972223634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:03,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:03,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:03,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:03,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:03,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972223634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:03,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:03,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:03,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:03,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-07T09:36:03,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:03,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. as already flushing 2024-11-07T09:36:03,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:03,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:03,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:03,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:03,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-07T09:36:03,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:03,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1730972223643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:03,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:03,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972223643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:03,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:03,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:03,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:03,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:03,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:03,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:03,714 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-07T09:36:03,715 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-07T09:36:03,717 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-07T09:36:03,717 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-07T09:36:03,718 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-07T09:36:03,718 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-07T09:36:03,719 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-07T09:36:03,719 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-07T09:36:03,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-07T09:36:03,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:03,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. as already flushing 2024-11-07T09:36:03,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:03,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:03,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:03,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:03,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:03,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:03,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972223843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:03,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33506 deadline: 1730972223843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:03,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:03,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:03,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:03,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:03,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972223844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:03,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:03,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:03,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:03,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:03,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:03,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:03,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:03,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1730972223853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:03,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:03,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:03,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:03,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:03,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972223853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:03,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:03,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:03,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:03,863 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/5529f3fb6df5466283cb3183ee25ffa8 2024-11-07T09:36:03,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/2f4002f0bcd8458381618626574ecbe8 is 50, key is test_row_0/B:col10/1730972163097/Put/seqid=0 2024-11-07T09:36:03,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741856_1032 (size=12001) 2024-11-07T09:36:03,900 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/2f4002f0bcd8458381618626574ecbe8 2024-11-07T09:36:03,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/57e9bf98e2c042c98ea9f31d8047168a is 50, key is test_row_0/C:col10/1730972163097/Put/seqid=0 2024-11-07T09:36:03,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741857_1033 (size=12001) 2024-11-07T09:36:03,929 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/57e9bf98e2c042c98ea9f31d8047168a 2024-11-07T09:36:03,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/5529f3fb6df5466283cb3183ee25ffa8 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/5529f3fb6df5466283cb3183ee25ffa8 2024-11-07T09:36:03,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-07T09:36:03,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:03,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. as already flushing 2024-11-07T09:36:03,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:03,951 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:03,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:03,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:03,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-07T09:36:03,960 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/5529f3fb6df5466283cb3183ee25ffa8, entries=200, sequenceid=115, filesize=14.0 K 2024-11-07T09:36:03,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/2f4002f0bcd8458381618626574ecbe8 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/2f4002f0bcd8458381618626574ecbe8 2024-11-07T09:36:03,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/2f4002f0bcd8458381618626574ecbe8, entries=150, sequenceid=115, filesize=11.7 K 2024-11-07T09:36:03,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/57e9bf98e2c042c98ea9f31d8047168a as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/57e9bf98e2c042c98ea9f31d8047168a 2024-11-07T09:36:04,000 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/57e9bf98e2c042c98ea9f31d8047168a, entries=150, sequenceid=115, filesize=11.7 K 2024-11-07T09:36:04,002 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 116dccc5098610cd38ab507cb3daad26 in 564ms, sequenceid=115, compaction requested=true 2024-11-07T09:36:04,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:04,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 116dccc5098610cd38ab507cb3daad26:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T09:36:04,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:04,003 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:36:04,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 116dccc5098610cd38ab507cb3daad26:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T09:36:04,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:04,003 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:36:04,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 116dccc5098610cd38ab507cb3daad26:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:36:04,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:36:04,007 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:36:04,008 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 116dccc5098610cd38ab507cb3daad26/B is initiating minor compaction (all files) 2024-11-07T09:36:04,008 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38481 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:36:04,008 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 116dccc5098610cd38ab507cb3daad26/A is initiating minor compaction (all files) 2024-11-07T09:36:04,008 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 116dccc5098610cd38ab507cb3daad26/B in TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:04,008 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 116dccc5098610cd38ab507cb3daad26/A in TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:04,008 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/86ec91df06724153b224b65a4d52b582, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/df505b5175df4652a5caa4fa80a960b8, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/5529f3fb6df5466283cb3183ee25ffa8] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp, totalSize=37.6 K 2024-11-07T09:36:04,009 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/76aa031fdcd34dd3949c9364ab4c5e49, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/d417061d29314c99946b5975d4b7d332, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/2f4002f0bcd8458381618626574ecbe8] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp, totalSize=35.3 K 2024-11-07T09:36:04,010 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86ec91df06724153b224b65a4d52b582, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1730972162012 2024-11-07T09:36:04,012 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting df505b5175df4652a5caa4fa80a960b8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1730972162708 2024-11-07T09:36:04,012 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 76aa031fdcd34dd3949c9364ab4c5e49, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1730972162012 2024-11-07T09:36:04,013 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5529f3fb6df5466283cb3183ee25ffa8, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1730972163097 2024-11-07T09:36:04,013 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting d417061d29314c99946b5975d4b7d332, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1730972162708 2024-11-07T09:36:04,014 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f4002f0bcd8458381618626574ecbe8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1730972163097 2024-11-07T09:36:04,041 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 116dccc5098610cd38ab507cb3daad26#B#compaction#21 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:04,042 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/68b762e4c8bf43ef9f1546c676664336 is 50, key is test_row_0/B:col10/1730972163097/Put/seqid=0 2024-11-07T09:36:04,059 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 116dccc5098610cd38ab507cb3daad26#A#compaction#22 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:04,060 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/6e734765090e429bbd775e73cf5777b2 is 50, key is test_row_0/A:col10/1730972163097/Put/seqid=0 2024-11-07T09:36:04,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741858_1034 (size=12241) 2024-11-07T09:36:04,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741859_1035 (size=12241) 2024-11-07T09:36:04,098 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/68b762e4c8bf43ef9f1546c676664336 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/68b762e4c8bf43ef9f1546c676664336 2024-11-07T09:36:04,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-07T09:36:04,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:04,107 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/6e734765090e429bbd775e73cf5777b2 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/6e734765090e429bbd775e73cf5777b2 2024-11-07T09:36:04,108 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 116dccc5098610cd38ab507cb3daad26 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-07T09:36:04,114 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 116dccc5098610cd38ab507cb3daad26/B of 116dccc5098610cd38ab507cb3daad26 into 68b762e4c8bf43ef9f1546c676664336(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:04,114 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:04,114 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., storeName=116dccc5098610cd38ab507cb3daad26/B, priority=13, startTime=1730972164003; duration=0sec 2024-11-07T09:36:04,114 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:36:04,114 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 116dccc5098610cd38ab507cb3daad26:B 2024-11-07T09:36:04,115 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:36:04,118 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:36:04,119 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 116dccc5098610cd38ab507cb3daad26/C is initiating minor compaction (all files) 2024-11-07T09:36:04,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/579dfee71e374768a9b117c8cd0aaf02 is 50, key is test_row_0/A:col10/1730972163524/Put/seqid=0 2024-11-07T09:36:04,119 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 116dccc5098610cd38ab507cb3daad26/C in TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:04,119 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/eb0801396c75482781c6dbfe4207462d, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/7f8174412ac14cf2b29f1bd83b5ffa6e, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/57e9bf98e2c042c98ea9f31d8047168a] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp, totalSize=35.3 K 2024-11-07T09:36:04,121 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting eb0801396c75482781c6dbfe4207462d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1730972162012 2024-11-07T09:36:04,122 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f8174412ac14cf2b29f1bd83b5ffa6e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1730972162708 2024-11-07T09:36:04,124 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 57e9bf98e2c042c98ea9f31d8047168a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1730972163097 2024-11-07T09:36:04,125 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 116dccc5098610cd38ab507cb3daad26/A of 116dccc5098610cd38ab507cb3daad26 into 6e734765090e429bbd775e73cf5777b2(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:04,126 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:04,126 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., storeName=116dccc5098610cd38ab507cb3daad26/A, priority=13, startTime=1730972164002; duration=0sec 2024-11-07T09:36:04,127 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:04,127 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 116dccc5098610cd38ab507cb3daad26:A 2024-11-07T09:36:04,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741860_1036 (size=12001) 2024-11-07T09:36:04,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 116dccc5098610cd38ab507cb3daad26 2024-11-07T09:36:04,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. as already flushing 2024-11-07T09:36:04,168 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 116dccc5098610cd38ab507cb3daad26#C#compaction#24 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:04,172 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/3290cd1e23284841a76ae1ed8b27a4c5 is 50, key is test_row_0/C:col10/1730972163097/Put/seqid=0 2024-11-07T09:36:04,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741861_1037 (size=12241) 2024-11-07T09:36:04,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:04,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33506 deadline: 1730972224203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:04,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:04,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:04,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:04,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:04,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972224203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:04,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:04,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:04,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:04,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:04,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1730972224206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:04,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:04,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:04,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:04,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:04,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972224209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:04,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:04,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:04,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:04,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:04,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972224212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:04,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:04,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:04,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:04,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:04,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:04,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33506 deadline: 1730972224323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:04,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972224324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:04,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:04,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:04,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:04,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:04,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:04,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1730972224324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:04,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972224325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:04,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:04,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:04,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:04,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:04,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:04,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:04,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:04,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:04,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:04,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:04,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972224334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:04,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:04,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:04,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:04,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-07T09:36:04,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:04,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972224543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:04,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:04,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33506 deadline: 1730972224544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:04,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:04,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:04,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:04,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:04,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972224544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:04,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:04,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:04,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:04,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:04,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:04,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1730972224544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:04,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972224544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:04,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:04,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:04,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:04,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:04,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:04,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:04,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:04,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:04,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:04,559 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/579dfee71e374768a9b117c8cd0aaf02 2024-11-07T09:36:04,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/8feb7caf9df54c37b7a2c0ae10d72c7a is 50, key is test_row_0/B:col10/1730972163524/Put/seqid=0 2024-11-07T09:36:04,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741862_1038 (size=12001) 2024-11-07T09:36:04,588 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/8feb7caf9df54c37b7a2c0ae10d72c7a 2024-11-07T09:36:04,616 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/3290cd1e23284841a76ae1ed8b27a4c5 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/3290cd1e23284841a76ae1ed8b27a4c5 2024-11-07T09:36:04,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/648a2109bc284ede837fa3fdae8bd8d9 is 50, key is test_row_0/C:col10/1730972163524/Put/seqid=0 2024-11-07T09:36:04,630 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 116dccc5098610cd38ab507cb3daad26/C of 116dccc5098610cd38ab507cb3daad26 into 3290cd1e23284841a76ae1ed8b27a4c5(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:04,630 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:04,630 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., storeName=116dccc5098610cd38ab507cb3daad26/C, priority=13, startTime=1730972164003; duration=0sec 2024-11-07T09:36:04,630 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:04,630 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 116dccc5098610cd38ab507cb3daad26:C 2024-11-07T09:36:04,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741863_1039 (size=12001) 2024-11-07T09:36:04,639 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/648a2109bc284ede837fa3fdae8bd8d9 2024-11-07T09:36:04,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/579dfee71e374768a9b117c8cd0aaf02 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/579dfee71e374768a9b117c8cd0aaf02 2024-11-07T09:36:04,664 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/579dfee71e374768a9b117c8cd0aaf02, entries=150, sequenceid=126, filesize=11.7 K 2024-11-07T09:36:04,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/8feb7caf9df54c37b7a2c0ae10d72c7a as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/8feb7caf9df54c37b7a2c0ae10d72c7a 2024-11-07T09:36:04,678 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/8feb7caf9df54c37b7a2c0ae10d72c7a, entries=150, sequenceid=126, filesize=11.7 K 2024-11-07T09:36:04,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/648a2109bc284ede837fa3fdae8bd8d9 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/648a2109bc284ede837fa3fdae8bd8d9 2024-11-07T09:36:04,690 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/648a2109bc284ede837fa3fdae8bd8d9, entries=150, sequenceid=126, filesize=11.7 K 2024-11-07T09:36:04,697 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 116dccc5098610cd38ab507cb3daad26 in 589ms, sequenceid=126, compaction requested=false 2024-11-07T09:36:04,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:04,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:04,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-07T09:36:04,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-07T09:36:04,703 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-07T09:36:04,703 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3710 sec 2024-11-07T09:36:04,707 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.3810 sec 2024-11-07T09:36:04,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 116dccc5098610cd38ab507cb3daad26 2024-11-07T09:36:04,870 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:36:04,871 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 116dccc5098610cd38ab507cb3daad26 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-07T09:36:04,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/699a85f190a240b787d2cdf01d3777a6 is 50, key is test_row_0/A:col10/1730972164870/Put/seqid=0 2024-11-07T09:36:04,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:04,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972224876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:04,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:04,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:04,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:04,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:04,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33506 deadline: 1730972224881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:04,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:04,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:04,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:04,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:04,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972224884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:04,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:04,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:04,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:04,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:04,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:04,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1730972224885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:04,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972224885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:04,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:04,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:04,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:04,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:04,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:04,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:04,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741864_1040 (size=12151) 2024-11-07T09:36:04,918 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/699a85f190a240b787d2cdf01d3777a6 2024-11-07T09:36:04,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/2e00e4f98372434aa2894da62117084f is 50, key is test_row_0/B:col10/1730972164870/Put/seqid=0 2024-11-07T09:36:04,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741865_1041 (size=12151) 2024-11-07T09:36:04,957 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/2e00e4f98372434aa2894da62117084f 2024-11-07T09:36:04,973 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/d16e972925ab4a41bb5b2532034a9a4b is 50, key is test_row_0/C:col10/1730972164870/Put/seqid=0 2024-11-07T09:36:04,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:04,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33506 deadline: 1730972224994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:05,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:05,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972224994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:05,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:05,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972224994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:05,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:05,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:05,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:05,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741866_1042 (size=12151) 2024-11-07T09:36:05,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:05,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:05,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1730972225003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:05,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972225003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:05,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:05,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:05,008 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/d16e972925ab4a41bb5b2532034a9a4b 2024-11-07T09:36:05,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/699a85f190a240b787d2cdf01d3777a6 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/699a85f190a240b787d2cdf01d3777a6 2024-11-07T09:36:05,034 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/699a85f190a240b787d2cdf01d3777a6, entries=150, sequenceid=159, filesize=11.9 K 2024-11-07T09:36:05,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/2e00e4f98372434aa2894da62117084f as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/2e00e4f98372434aa2894da62117084f 2024-11-07T09:36:05,044 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/2e00e4f98372434aa2894da62117084f, entries=150, sequenceid=159, filesize=11.9 K 2024-11-07T09:36:05,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/d16e972925ab4a41bb5b2532034a9a4b as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/d16e972925ab4a41bb5b2532034a9a4b 2024-11-07T09:36:05,062 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/d16e972925ab4a41bb5b2532034a9a4b, entries=150, sequenceid=159, filesize=11.9 K 2024-11-07T09:36:05,064 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=26.84 KB/27480 for 116dccc5098610cd38ab507cb3daad26 in 193ms, sequenceid=159, compaction requested=true 2024-11-07T09:36:05,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:05,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 116dccc5098610cd38ab507cb3daad26:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T09:36:05,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:05,064 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:36:05,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 116dccc5098610cd38ab507cb3daad26:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T09:36:05,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:05,065 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:36:05,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 116dccc5098610cd38ab507cb3daad26:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:36:05,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:36:05,067 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:36:05,067 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 116dccc5098610cd38ab507cb3daad26/A is initiating minor compaction (all files) 2024-11-07T09:36:05,067 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 116dccc5098610cd38ab507cb3daad26/A in TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:05,068 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/6e734765090e429bbd775e73cf5777b2, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/579dfee71e374768a9b117c8cd0aaf02, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/699a85f190a240b787d2cdf01d3777a6] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp, totalSize=35.5 K 2024-11-07T09:36:05,068 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:36:05,069 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 116dccc5098610cd38ab507cb3daad26/B is initiating minor compaction (all files) 2024-11-07T09:36:05,069 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 116dccc5098610cd38ab507cb3daad26/B in TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:05,069 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/68b762e4c8bf43ef9f1546c676664336, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/8feb7caf9df54c37b7a2c0ae10d72c7a, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/2e00e4f98372434aa2894da62117084f] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp, totalSize=35.5 K 2024-11-07T09:36:05,070 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e734765090e429bbd775e73cf5777b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1730972163097 2024-11-07T09:36:05,070 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 68b762e4c8bf43ef9f1546c676664336, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1730972163097 2024-11-07T09:36:05,071 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 8feb7caf9df54c37b7a2c0ae10d72c7a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1730972163511 2024-11-07T09:36:05,071 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 579dfee71e374768a9b117c8cd0aaf02, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1730972163511 2024-11-07T09:36:05,071 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e00e4f98372434aa2894da62117084f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1730972164197 2024-11-07T09:36:05,072 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 699a85f190a240b787d2cdf01d3777a6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1730972164197 2024-11-07T09:36:05,093 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 116dccc5098610cd38ab507cb3daad26#B#compaction#30 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:05,093 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/0379cd87cfca4b0bb7142ebda87b63c4 is 50, key is test_row_0/B:col10/1730972164870/Put/seqid=0 2024-11-07T09:36:05,099 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 116dccc5098610cd38ab507cb3daad26#A#compaction#31 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:05,100 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/96eaa21aed454bfeb10d3581644e6a5d is 50, key is test_row_0/A:col10/1730972164870/Put/seqid=0 2024-11-07T09:36:05,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741867_1043 (size=12493) 2024-11-07T09:36:05,131 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/0379cd87cfca4b0bb7142ebda87b63c4 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/0379cd87cfca4b0bb7142ebda87b63c4 2024-11-07T09:36:05,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741868_1044 (size=12493) 2024-11-07T09:36:05,143 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/96eaa21aed454bfeb10d3581644e6a5d as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/96eaa21aed454bfeb10d3581644e6a5d 2024-11-07T09:36:05,145 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 116dccc5098610cd38ab507cb3daad26/B of 116dccc5098610cd38ab507cb3daad26 into 0379cd87cfca4b0bb7142ebda87b63c4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:05,145 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:05,145 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., storeName=116dccc5098610cd38ab507cb3daad26/B, priority=13, startTime=1730972165064; duration=0sec 2024-11-07T09:36:05,145 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:36:05,145 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 116dccc5098610cd38ab507cb3daad26:B 2024-11-07T09:36:05,146 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:36:05,150 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:36:05,150 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 116dccc5098610cd38ab507cb3daad26/C is initiating minor compaction (all files) 2024-11-07T09:36:05,150 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 116dccc5098610cd38ab507cb3daad26/C in TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:05,150 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/3290cd1e23284841a76ae1ed8b27a4c5, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/648a2109bc284ede837fa3fdae8bd8d9, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/d16e972925ab4a41bb5b2532034a9a4b] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp, totalSize=35.5 K 2024-11-07T09:36:05,151 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 3290cd1e23284841a76ae1ed8b27a4c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1730972163097 2024-11-07T09:36:05,152 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 648a2109bc284ede837fa3fdae8bd8d9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1730972163511 2024-11-07T09:36:05,154 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting d16e972925ab4a41bb5b2532034a9a4b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1730972164197 2024-11-07T09:36:05,157 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 116dccc5098610cd38ab507cb3daad26/A of 116dccc5098610cd38ab507cb3daad26 into 96eaa21aed454bfeb10d3581644e6a5d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:05,157 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:05,158 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., storeName=116dccc5098610cd38ab507cb3daad26/A, priority=13, startTime=1730972165064; duration=0sec 2024-11-07T09:36:05,158 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:05,158 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 116dccc5098610cd38ab507cb3daad26:A 2024-11-07T09:36:05,178 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 116dccc5098610cd38ab507cb3daad26#C#compaction#32 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:05,178 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/dcfa1fbedc1f4e2a9e97d0f19cd6aa61 is 50, key is test_row_0/C:col10/1730972164870/Put/seqid=0 2024-11-07T09:36:05,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741869_1045 (size=12493) 2024-11-07T09:36:05,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 116dccc5098610cd38ab507cb3daad26 2024-11-07T09:36:05,220 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:36:05,220 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 116dccc5098610cd38ab507cb3daad26 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T09:36:05,231 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/67e95a76c13d4b0cb4e869683a00b74c is 50, key is test_row_0/A:col10/1730972165219/Put/seqid=0 2024-11-07T09:36:05,233 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/dcfa1fbedc1f4e2a9e97d0f19cd6aa61 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/dcfa1fbedc1f4e2a9e97d0f19cd6aa61 2024-11-07T09:36:05,245 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 116dccc5098610cd38ab507cb3daad26/C of 116dccc5098610cd38ab507cb3daad26 into dcfa1fbedc1f4e2a9e97d0f19cd6aa61(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:05,245 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:05,246 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., storeName=116dccc5098610cd38ab507cb3daad26/C, priority=13, startTime=1730972165065; duration=0sec 2024-11-07T09:36:05,246 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:05,246 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 116dccc5098610cd38ab507cb3daad26:C 2024-11-07T09:36:05,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:05,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972225260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:05,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:05,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741870_1046 (size=12151) 2024-11-07T09:36:05,276 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/67e95a76c13d4b0cb4e869683a00b74c 2024-11-07T09:36:05,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:05,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972225267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:05,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:05,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:05,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972225273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:05,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:05,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/baaba6cc6572451c9152cd9ad571b5b3 is 50, key is test_row_0/B:col10/1730972165219/Put/seqid=0 2024-11-07T09:36:05,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:05,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33506 deadline: 1730972225288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:05,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:05,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1730972225287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:05,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:05,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:05,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741871_1047 (size=12151) 2024-11-07T09:36:05,302 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/baaba6cc6572451c9152cd9ad571b5b3 2024-11-07T09:36:05,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/5f519eb6cc6d433f8cb529c0bbc9bd06 is 50, key is test_row_0/C:col10/1730972165219/Put/seqid=0 2024-11-07T09:36:05,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741872_1048 (size=12151) 2024-11-07T09:36:05,351 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/5f519eb6cc6d433f8cb529c0bbc9bd06 2024-11-07T09:36:05,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/67e95a76c13d4b0cb4e869683a00b74c as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/67e95a76c13d4b0cb4e869683a00b74c 2024-11-07T09:36:05,371 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/67e95a76c13d4b0cb4e869683a00b74c, entries=150, sequenceid=173, filesize=11.9 K 2024-11-07T09:36:05,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/baaba6cc6572451c9152cd9ad571b5b3 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/baaba6cc6572451c9152cd9ad571b5b3 2024-11-07T09:36:05,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:05,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972225373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:05,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:05,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/baaba6cc6572451c9152cd9ad571b5b3, entries=150, sequenceid=173, filesize=11.9 K 2024-11-07T09:36:05,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/5f519eb6cc6d433f8cb529c0bbc9bd06 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/5f519eb6cc6d433f8cb529c0bbc9bd06 2024-11-07T09:36:05,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:05,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972225393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:05,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:05,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:05,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972225393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:05,396 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/5f519eb6cc6d433f8cb529c0bbc9bd06, entries=150, sequenceid=173, filesize=11.9 K 2024-11-07T09:36:05,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:05,398 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=154.31 KB/158010 for 116dccc5098610cd38ab507cb3daad26 in 178ms, sequenceid=173, compaction requested=false 2024-11-07T09:36:05,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:05,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 116dccc5098610cd38ab507cb3daad26 2024-11-07T09:36:05,408 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:36:05,408 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 116dccc5098610cd38ab507cb3daad26 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-07T09:36:05,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/9ae015902c314504b07dbe02b3621147 is 50, key is test_row_0/A:col10/1730972165405/Put/seqid=0 2024-11-07T09:36:05,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741873_1049 (size=16931) 2024-11-07T09:36:05,430 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/9ae015902c314504b07dbe02b3621147 2024-11-07T09:36:05,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:05,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33506 deadline: 1730972225432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:05,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:05,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:05,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1730972225438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:05,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:05,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/a4bfbfe2c6434b55a7b956bda54efd1e is 50, key is test_row_0/B:col10/1730972165405/Put/seqid=0 2024-11-07T09:36:05,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741874_1050 (size=12151) 2024-11-07T09:36:05,474 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/a4bfbfe2c6434b55a7b956bda54efd1e 2024-11-07T09:36:05,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-07T09:36:05,475 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:36:05,475 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T09:36:05,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-07T09:36:05,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-07T09:36:05,479 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T09:36:05,480 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T09:36:05,480 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T09:36:05,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/a8b5e47306714288802dd6502b0a519d is 50, key is test_row_0/C:col10/1730972165405/Put/seqid=0 2024-11-07T09:36:05,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741875_1051 (size=12151) 2024-11-07T09:36:05,504 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/a8b5e47306714288802dd6502b0a519d 2024-11-07T09:36:05,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/9ae015902c314504b07dbe02b3621147 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/9ae015902c314504b07dbe02b3621147 2024-11-07T09:36:05,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/9ae015902c314504b07dbe02b3621147, entries=250, sequenceid=202, filesize=16.5 K 2024-11-07T09:36:05,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/a4bfbfe2c6434b55a7b956bda54efd1e as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/a4bfbfe2c6434b55a7b956bda54efd1e 2024-11-07T09:36:05,541 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/a4bfbfe2c6434b55a7b956bda54efd1e, entries=150, sequenceid=202, filesize=11.9 K 2024-11-07T09:36:05,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/a8b5e47306714288802dd6502b0a519d as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/a8b5e47306714288802dd6502b0a519d 2024-11-07T09:36:05,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:05,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33506 deadline: 1730972225544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:05,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:05,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:05,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1730972225553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:05,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:05,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/a8b5e47306714288802dd6502b0a519d, entries=150, sequenceid=202, filesize=11.9 K 2024-11-07T09:36:05,566 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 116dccc5098610cd38ab507cb3daad26 in 158ms, sequenceid=202, compaction requested=true 2024-11-07T09:36:05,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:05,567 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:36:05,570 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41575 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:36:05,570 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 116dccc5098610cd38ab507cb3daad26/A is initiating minor compaction (all files) 2024-11-07T09:36:05,570 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 116dccc5098610cd38ab507cb3daad26/A in TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:05,570 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/96eaa21aed454bfeb10d3581644e6a5d, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/67e95a76c13d4b0cb4e869683a00b74c, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/9ae015902c314504b07dbe02b3621147] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp, totalSize=40.6 K 2024-11-07T09:36:05,571 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 96eaa21aed454bfeb10d3581644e6a5d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1730972164197 2024-11-07T09:36:05,571 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 67e95a76c13d4b0cb4e869683a00b74c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1730972165212 2024-11-07T09:36:05,572 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ae015902c314504b07dbe02b3621147, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1730972165250 2024-11-07T09:36:05,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 116dccc5098610cd38ab507cb3daad26:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T09:36:05,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:05,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 116dccc5098610cd38ab507cb3daad26:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T09:36:05,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:05,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 116dccc5098610cd38ab507cb3daad26:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:36:05,573 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:36:05,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:36:05,577 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:36:05,577 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 116dccc5098610cd38ab507cb3daad26/B is initiating minor compaction (all files) 2024-11-07T09:36:05,577 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 116dccc5098610cd38ab507cb3daad26/B in TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:05,577 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/0379cd87cfca4b0bb7142ebda87b63c4, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/baaba6cc6572451c9152cd9ad571b5b3, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/a4bfbfe2c6434b55a7b956bda54efd1e] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp, totalSize=35.9 K 2024-11-07T09:36:05,579 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0379cd87cfca4b0bb7142ebda87b63c4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1730972164197 2024-11-07T09:36:05,579 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting baaba6cc6572451c9152cd9ad571b5b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1730972165212 2024-11-07T09:36:05,582 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4bfbfe2c6434b55a7b956bda54efd1e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1730972165267 2024-11-07T09:36:05,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-07T09:36:05,594 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 116dccc5098610cd38ab507cb3daad26#A#compaction#39 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:05,595 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/2819a6c970ee43b4856f9a9af12a2ad9 is 50, key is test_row_0/A:col10/1730972165405/Put/seqid=0 2024-11-07T09:36:05,598 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 116dccc5098610cd38ab507cb3daad26#B#compaction#40 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:05,598 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/75b3fe5ae2d54d34838c56b7dca1eee4 is 50, key is test_row_0/B:col10/1730972165405/Put/seqid=0 2024-11-07T09:36:05,601 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:36:05,602 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 116dccc5098610cd38ab507cb3daad26 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T09:36:05,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 116dccc5098610cd38ab507cb3daad26 2024-11-07T09:36:05,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/345f728ae70d4b7db7322a9868e4b5ed is 50, key is test_row_0/A:col10/1730972165598/Put/seqid=0 2024-11-07T09:36:05,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741876_1052 (size=12595) 2024-11-07T09:36:05,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-07T09:36:05,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:05,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. as already flushing 2024-11-07T09:36:05,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:05,637 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:05,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:05,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:05,642 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/2819a6c970ee43b4856f9a9af12a2ad9 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/2819a6c970ee43b4856f9a9af12a2ad9 2024-11-07T09:36:05,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741877_1053 (size=12595) 2024-11-07T09:36:05,661 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/75b3fe5ae2d54d34838c56b7dca1eee4 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/75b3fe5ae2d54d34838c56b7dca1eee4 2024-11-07T09:36:05,670 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 116dccc5098610cd38ab507cb3daad26/A of 116dccc5098610cd38ab507cb3daad26 into 2819a6c970ee43b4856f9a9af12a2ad9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:05,670 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:05,670 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., storeName=116dccc5098610cd38ab507cb3daad26/A, priority=13, startTime=1730972165567; duration=0sec 2024-11-07T09:36:05,670 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:36:05,671 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 116dccc5098610cd38ab507cb3daad26:A 2024-11-07T09:36:05,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:05,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972225664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:05,672 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 116dccc5098610cd38ab507cb3daad26/B of 116dccc5098610cd38ab507cb3daad26 into 75b3fe5ae2d54d34838c56b7dca1eee4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:05,672 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:05,672 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., storeName=116dccc5098610cd38ab507cb3daad26/B, priority=13, startTime=1730972165572; duration=0sec 2024-11-07T09:36:05,672 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:05,672 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 116dccc5098610cd38ab507cb3daad26:B 2024-11-07T09:36:05,672 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:36:05,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:05,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972225668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:05,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,673 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:05,673 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,673 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:05,675 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:36:05,675 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 116dccc5098610cd38ab507cb3daad26/C is initiating minor compaction (all files) 2024-11-07T09:36:05,675 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 116dccc5098610cd38ab507cb3daad26/C in TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:05,675 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/dcfa1fbedc1f4e2a9e97d0f19cd6aa61, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/5f519eb6cc6d433f8cb529c0bbc9bd06, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/a8b5e47306714288802dd6502b0a519d] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp, totalSize=35.9 K 2024-11-07T09:36:05,676 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting dcfa1fbedc1f4e2a9e97d0f19cd6aa61, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1730972164197 2024-11-07T09:36:05,676 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f519eb6cc6d433f8cb529c0bbc9bd06, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1730972165212 2024-11-07T09:36:05,677 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting a8b5e47306714288802dd6502b0a519d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1730972165267 2024-11-07T09:36:05,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:05,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972225672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:05,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:05,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741878_1054 (size=14541) 2024-11-07T09:36:05,684 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/345f728ae70d4b7db7322a9868e4b5ed 2024-11-07T09:36:05,704 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 116dccc5098610cd38ab507cb3daad26#C#compaction#42 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:05,705 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/d3330ca2c9be45c09b30e37286642983 is 50, key is test_row_0/C:col10/1730972165405/Put/seqid=0 2024-11-07T09:36:05,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/7c40733fb8f1444990b9dee089d14cfc is 50, key is test_row_0/B:col10/1730972165598/Put/seqid=0 2024-11-07T09:36:05,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741879_1055 (size=12595) 2024-11-07T09:36:05,735 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/d3330ca2c9be45c09b30e37286642983 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/d3330ca2c9be45c09b30e37286642983 2024-11-07T09:36:05,746 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 116dccc5098610cd38ab507cb3daad26/C of 116dccc5098610cd38ab507cb3daad26 into d3330ca2c9be45c09b30e37286642983(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:05,746 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:05,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741880_1056 (size=12151) 2024-11-07T09:36:05,748 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., storeName=116dccc5098610cd38ab507cb3daad26/C, priority=13, startTime=1730972165573; duration=0sec 2024-11-07T09:36:05,748 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:05,748 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 116dccc5098610cd38ab507cb3daad26:C 2024-11-07T09:36:05,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:05,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33506 deadline: 1730972225754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:05,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:05,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:05,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1730972225763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:05,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:05,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:05,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972225783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:05,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:05,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:05,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972225783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:05,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:05,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:05,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972225783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:05,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:05,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-07T09:36:05,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:05,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. as already flushing 2024-11-07T09:36:05,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:05,792 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:05,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:05,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:05,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-07T09:36:05,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-07T09:36:05,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:05,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. as already flushing 2024-11-07T09:36:05,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:05,947 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:05,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:05,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:05,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:05,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972225994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:05,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:05,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:05,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972225994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:05,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:05,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972225994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:05,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:05,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:05,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:06,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33506 deadline: 1730972226064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:06,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:06,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:06,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:06,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:06,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1730972226072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:06,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:06,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:06,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:06,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-07T09:36:06,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:06,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. as already flushing 2024-11-07T09:36:06,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:06,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:06,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:06,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:06,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-07T09:36:06,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/7c40733fb8f1444990b9dee089d14cfc 2024-11-07T09:36:06,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/49986d0a00324c30a09dc20bf790b14a is 50, key is test_row_0/C:col10/1730972165598/Put/seqid=0 2024-11-07T09:36:06,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741881_1057 (size=12151) 2024-11-07T09:36:06,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-07T09:36:06,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:06,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. as already flushing 2024-11-07T09:36:06,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:06,257 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:06,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:06,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:06,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:06,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:06,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:06,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972226302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:06,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972226303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:06,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972226303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:06,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:06,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:06,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:06,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:06,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:06,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:06,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:06,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:06,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:06,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-07T09:36:06,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:06,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. as already flushing 2024-11-07T09:36:06,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:06,411 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:06,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:06,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:06,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-07T09:36:06,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:06,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. as already flushing 2024-11-07T09:36:06,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:06,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:06,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:06,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:06,569 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/49986d0a00324c30a09dc20bf790b14a 2024-11-07T09:36:06,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:06,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33506 deadline: 1730972226573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:06,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/345f728ae70d4b7db7322a9868e4b5ed as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/345f728ae70d4b7db7322a9868e4b5ed 2024-11-07T09:36:06,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:06,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:06,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:06,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:06,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1730972226583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:06,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:06,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:06,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:06,586 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/345f728ae70d4b7db7322a9868e4b5ed, entries=200, sequenceid=213, filesize=14.2 K 2024-11-07T09:36:06,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/7c40733fb8f1444990b9dee089d14cfc as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/7c40733fb8f1444990b9dee089d14cfc 2024-11-07T09:36:06,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/7c40733fb8f1444990b9dee089d14cfc, entries=150, sequenceid=213, filesize=11.9 K 2024-11-07T09:36:06,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/49986d0a00324c30a09dc20bf790b14a as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/49986d0a00324c30a09dc20bf790b14a 2024-11-07T09:36:06,606 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/49986d0a00324c30a09dc20bf790b14a, entries=150, sequenceid=213, filesize=11.9 K 2024-11-07T09:36:06,608 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 116dccc5098610cd38ab507cb3daad26 in 1006ms, sequenceid=213, compaction requested=false 2024-11-07T09:36:06,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:06,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-07T09:36:06,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-07T09:36:06,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:06,721 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 116dccc5098610cd38ab507cb3daad26 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T09:36:06,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/0e1769cdf0174fcb9ae9ffbc518ee4a6 is 50, key is test_row_0/A:col10/1730972165666/Put/seqid=0 2024-11-07T09:36:06,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741882_1058 (size=12151) 2024-11-07T09:36:06,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 116dccc5098610cd38ab507cb3daad26 2024-11-07T09:36:06,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. as already flushing 2024-11-07T09:36:06,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:06,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972226832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:06,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:06,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:06,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:06,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:06,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972226834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:06,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:06,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:06,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:06,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:06,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972226834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:06,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:06,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:06,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:06,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:06,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972226943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:06,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:06,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:06,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:06,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:06,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972226952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:06,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:06,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:06,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:06,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:06,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972226953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:06,954 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:06,954 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:06,954 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:07,149 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/0e1769cdf0174fcb9ae9ffbc518ee4a6 2024-11-07T09:36:07,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:07,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972227154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:07,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:07,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:07,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:07,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/4aedce6aedcb488b93999dc006c440f0 is 50, key is test_row_0/B:col10/1730972165666/Put/seqid=0 2024-11-07T09:36:07,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:07,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972227163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:07,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:07,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:07,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:07,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:07,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972227163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:07,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:07,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:07,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:07,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741883_1059 (size=12151) 2024-11-07T09:36:07,180 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/4aedce6aedcb488b93999dc006c440f0 2024-11-07T09:36:07,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/13972d40cb9e40b394430d9e41e334e5 is 50, key is test_row_0/C:col10/1730972165666/Put/seqid=0 2024-11-07T09:36:07,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741884_1060 (size=12151) 2024-11-07T09:36:07,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:07,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972227463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:07,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:07,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:07,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:07,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:07,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972227473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:07,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:07,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:07,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:07,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:07,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972227473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:07,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:07,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:07,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:07,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:07,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33506 deadline: 1730972227593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:07,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:07,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1730972227593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:07,598 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 9f1eb259a29b,42895,1730972153970 for 'test_row_2' in 116dccc5098610cd38ab507cb3daad26 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T09:36:07,598 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 9f1eb259a29b,42895,1730972153970 for 'test_row_1' in 116dccc5098610cd38ab507cb3daad26 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T09:36:07,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:07,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:07,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:07,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:07,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:07,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:07,612 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/13972d40cb9e40b394430d9e41e334e5 2024-11-07T09:36:07,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/0e1769cdf0174fcb9ae9ffbc518ee4a6 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/0e1769cdf0174fcb9ae9ffbc518ee4a6 2024-11-07T09:36:07,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-07T09:36:07,626 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/0e1769cdf0174fcb9ae9ffbc518ee4a6, entries=150, sequenceid=241, filesize=11.9 K 2024-11-07T09:36:07,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/4aedce6aedcb488b93999dc006c440f0 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/4aedce6aedcb488b93999dc006c440f0 2024-11-07T09:36:07,635 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/4aedce6aedcb488b93999dc006c440f0, entries=150, sequenceid=241, filesize=11.9 K 2024-11-07T09:36:07,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/13972d40cb9e40b394430d9e41e334e5 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/13972d40cb9e40b394430d9e41e334e5 2024-11-07T09:36:07,644 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/13972d40cb9e40b394430d9e41e334e5, entries=150, sequenceid=241, filesize=11.9 K 2024-11-07T09:36:07,645 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 116dccc5098610cd38ab507cb3daad26 in 924ms, sequenceid=241, compaction requested=true 2024-11-07T09:36:07,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:07,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:07,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-07T09:36:07,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-07T09:36:07,651 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-07T09:36:07,651 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1680 sec 2024-11-07T09:36:07,654 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 2.1770 sec 2024-11-07T09:36:07,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 116dccc5098610cd38ab507cb3daad26 2024-11-07T09:36:07,977 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:36:07,977 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 116dccc5098610cd38ab507cb3daad26 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-07T09:36:07,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/8524df10b8d141d6839c633dc70bd3ad is 50, key is test_row_0/A:col10/1730972167975/Put/seqid=0 2024-11-07T09:36:08,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741885_1061 (size=12151) 2024-11-07T09:36:08,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:08,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972228028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:08,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:08,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:08,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:08,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972228029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:08,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:08,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:08,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:08,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:08,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:08,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972228030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:08,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:08,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:08,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:08,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:08,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972228143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:08,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:08,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:08,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:08,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:08,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972228143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:08,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:08,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:08,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:08,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:08,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972228143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:08,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:08,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:08,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:08,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:08,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972228353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:08,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:08,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:08,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972228353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:08,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:08,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:08,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:08,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:08,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:08,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:08,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972228353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:08,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:08,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:08,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:08,402 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/8524df10b8d141d6839c633dc70bd3ad 2024-11-07T09:36:08,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/0c0a839cf4784be88a36b01af7ef459a is 50, key is test_row_0/B:col10/1730972167975/Put/seqid=0 2024-11-07T09:36:08,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741886_1062 (size=12151) 2024-11-07T09:36:08,428 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/0c0a839cf4784be88a36b01af7ef459a 2024-11-07T09:36:08,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/5fe6ec9030bf4d419f307d59dbfdc007 is 50, key is test_row_0/C:col10/1730972167975/Put/seqid=0 2024-11-07T09:36:08,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741887_1063 (size=12151) 2024-11-07T09:36:08,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:08,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972228663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:08,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:08,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972228663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:08,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:08,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:08,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:08,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:08,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:08,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972228663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:08,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:08,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:08,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:08,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:08,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:08,856 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/5fe6ec9030bf4d419f307d59dbfdc007 2024-11-07T09:36:08,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/8524df10b8d141d6839c633dc70bd3ad as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/8524df10b8d141d6839c633dc70bd3ad 2024-11-07T09:36:08,870 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/8524df10b8d141d6839c633dc70bd3ad, entries=150, sequenceid=254, filesize=11.9 K 2024-11-07T09:36:08,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/0c0a839cf4784be88a36b01af7ef459a as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/0c0a839cf4784be88a36b01af7ef459a 2024-11-07T09:36:08,883 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/0c0a839cf4784be88a36b01af7ef459a, entries=150, sequenceid=254, filesize=11.9 K 2024-11-07T09:36:08,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/5fe6ec9030bf4d419f307d59dbfdc007 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/5fe6ec9030bf4d419f307d59dbfdc007 2024-11-07T09:36:08,891 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/5fe6ec9030bf4d419f307d59dbfdc007, entries=150, sequenceid=254, filesize=11.9 K 2024-11-07T09:36:08,892 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 116dccc5098610cd38ab507cb3daad26 in 915ms, sequenceid=254, compaction requested=true 2024-11-07T09:36:08,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:08,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 116dccc5098610cd38ab507cb3daad26:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T09:36:08,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:08,893 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T09:36:08,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 116dccc5098610cd38ab507cb3daad26:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T09:36:08,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:36:08,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 116dccc5098610cd38ab507cb3daad26:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:36:08,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T09:36:08,893 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T09:36:08,895 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T09:36:08,896 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 116dccc5098610cd38ab507cb3daad26/B is initiating minor compaction (all files) 2024-11-07T09:36:08,896 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 116dccc5098610cd38ab507cb3daad26/B in TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:08,896 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/75b3fe5ae2d54d34838c56b7dca1eee4, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/7c40733fb8f1444990b9dee089d14cfc, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/4aedce6aedcb488b93999dc006c440f0, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/0c0a839cf4784be88a36b01af7ef459a] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp, totalSize=47.9 K 2024-11-07T09:36:08,896 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51438 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T09:36:08,896 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 116dccc5098610cd38ab507cb3daad26/A is initiating minor compaction (all files) 2024-11-07T09:36:08,896 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 116dccc5098610cd38ab507cb3daad26/A in TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:08,896 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/2819a6c970ee43b4856f9a9af12a2ad9, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/345f728ae70d4b7db7322a9868e4b5ed, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/0e1769cdf0174fcb9ae9ffbc518ee4a6, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/8524df10b8d141d6839c633dc70bd3ad] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp, totalSize=50.2 K 2024-11-07T09:36:08,897 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 75b3fe5ae2d54d34838c56b7dca1eee4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1730972165267 2024-11-07T09:36:08,897 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2819a6c970ee43b4856f9a9af12a2ad9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1730972165267 2024-11-07T09:36:08,897 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c40733fb8f1444990b9dee089d14cfc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1730972165421 2024-11-07T09:36:08,898 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 345f728ae70d4b7db7322a9868e4b5ed, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1730972165421 2024-11-07T09:36:08,898 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 4aedce6aedcb488b93999dc006c440f0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1730972165658 2024-11-07T09:36:08,898 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c0a839cf4784be88a36b01af7ef459a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1730972166832 2024-11-07T09:36:08,899 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e1769cdf0174fcb9ae9ffbc518ee4a6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1730972165658 2024-11-07T09:36:08,899 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8524df10b8d141d6839c633dc70bd3ad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1730972166832 2024-11-07T09:36:08,919 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 116dccc5098610cd38ab507cb3daad26#B#compaction#51 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:08,920 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/6951fbd9c41c4a70b77f0c14961dd8bc is 50, key is test_row_0/B:col10/1730972167975/Put/seqid=0 2024-11-07T09:36:08,920 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 116dccc5098610cd38ab507cb3daad26#A#compaction#52 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:08,921 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/c26bf73380a5467b9d5ac64f99b71463 is 50, key is test_row_0/A:col10/1730972167975/Put/seqid=0 2024-11-07T09:36:08,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741889_1065 (size=12731) 2024-11-07T09:36:08,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741888_1064 (size=12731) 2024-11-07T09:36:08,948 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/6951fbd9c41c4a70b77f0c14961dd8bc as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/6951fbd9c41c4a70b77f0c14961dd8bc 2024-11-07T09:36:08,958 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 116dccc5098610cd38ab507cb3daad26/B of 116dccc5098610cd38ab507cb3daad26 into 6951fbd9c41c4a70b77f0c14961dd8bc(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:08,958 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:08,958 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., storeName=116dccc5098610cd38ab507cb3daad26/B, priority=12, startTime=1730972168893; duration=0sec 2024-11-07T09:36:08,958 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:36:08,959 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 116dccc5098610cd38ab507cb3daad26:B 2024-11-07T09:36:08,959 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T09:36:08,960 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T09:36:08,961 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 116dccc5098610cd38ab507cb3daad26/C is initiating minor compaction (all files) 2024-11-07T09:36:08,961 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 116dccc5098610cd38ab507cb3daad26/C in TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:08,961 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/d3330ca2c9be45c09b30e37286642983, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/49986d0a00324c30a09dc20bf790b14a, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/13972d40cb9e40b394430d9e41e334e5, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/5fe6ec9030bf4d419f307d59dbfdc007] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp, totalSize=47.9 K 2024-11-07T09:36:08,961 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting d3330ca2c9be45c09b30e37286642983, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1730972165267 2024-11-07T09:36:08,962 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 49986d0a00324c30a09dc20bf790b14a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1730972165421 2024-11-07T09:36:08,962 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 13972d40cb9e40b394430d9e41e334e5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1730972165658 2024-11-07T09:36:08,963 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fe6ec9030bf4d419f307d59dbfdc007, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1730972166832 2024-11-07T09:36:08,983 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 116dccc5098610cd38ab507cb3daad26#C#compaction#53 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:08,984 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/970c7ecb9b64484585c148efd66bb851 is 50, key is test_row_0/C:col10/1730972167975/Put/seqid=0 2024-11-07T09:36:08,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741890_1066 (size=12731) 2024-11-07T09:36:09,007 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/970c7ecb9b64484585c148efd66bb851 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/970c7ecb9b64484585c148efd66bb851 2024-11-07T09:36:09,015 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 116dccc5098610cd38ab507cb3daad26/C of 116dccc5098610cd38ab507cb3daad26 into 970c7ecb9b64484585c148efd66bb851(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:09,015 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:09,015 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., storeName=116dccc5098610cd38ab507cb3daad26/C, priority=12, startTime=1730972168893; duration=0sec 2024-11-07T09:36:09,015 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:09,016 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 116dccc5098610cd38ab507cb3daad26:C 2024-11-07T09:36:09,176 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:36:09,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 116dccc5098610cd38ab507cb3daad26 2024-11-07T09:36:09,176 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 116dccc5098610cd38ab507cb3daad26 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T09:36:09,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/9973be5373334892a5704b8f0a63f8b4 is 50, key is test_row_0/A:col10/1730972168029/Put/seqid=0 2024-11-07T09:36:09,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741891_1067 (size=14741) 2024-11-07T09:36:09,198 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/9973be5373334892a5704b8f0a63f8b4 2024-11-07T09:36:09,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:09,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972229200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:09,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:09,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:09,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:09,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:09,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972229201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:09,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:09,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:09,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:09,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:09,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972229202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:09,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:09,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:09,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:09,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/dffd9b43a7cd49f0936e35152fa012dd is 50, key is test_row_0/B:col10/1730972168029/Put/seqid=0 2024-11-07T09:36:09,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741892_1068 (size=12301) 2024-11-07T09:36:09,219 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/dffd9b43a7cd49f0936e35152fa012dd 2024-11-07T09:36:09,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/f40511bd2dc24376ad090c47a8d5f353 is 50, key is test_row_0/C:col10/1730972168029/Put/seqid=0 2024-11-07T09:36:09,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741893_1069 (size=12301) 2024-11-07T09:36:09,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:09,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972229313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:09,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:09,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:09,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:09,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:09,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972229313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:09,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:09,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:09,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972229313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:09,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:09,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:09,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:09,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:09,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:09,345 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/c26bf73380a5467b9d5ac64f99b71463 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/c26bf73380a5467b9d5ac64f99b71463 2024-11-07T09:36:09,354 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 116dccc5098610cd38ab507cb3daad26/A of 116dccc5098610cd38ab507cb3daad26 into c26bf73380a5467b9d5ac64f99b71463(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:09,354 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:09,354 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., storeName=116dccc5098610cd38ab507cb3daad26/A, priority=12, startTime=1730972168893; duration=0sec 2024-11-07T09:36:09,354 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:09,354 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 116dccc5098610cd38ab507cb3daad26:A 2024-11-07T09:36:09,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:09,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:09,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972229523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:09,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972229523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:09,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:09,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:09,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:09,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:09,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:09,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972229524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:09,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:09,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:09,531 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:09,531 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:09,531 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:09,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:09,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1730972229622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:09,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:09,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33506 deadline: 1730972229623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:09,628 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 9f1eb259a29b,42895,1730972153970 for 'test_row_1' in 116dccc5098610cd38ab507cb3daad26 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T09:36:09,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:09,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:09,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:09,629 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 9f1eb259a29b,42895,1730972153970 for 'test_row_2' in 116dccc5098610cd38ab507cb3daad26 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T09:36:09,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:09,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:09,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:09,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-07T09:36:09,643 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:36:09,644 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T09:36:09,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-07T09:36:09,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-07T09:36:09,648 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T09:36:09,649 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T09:36:09,649 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T09:36:09,660 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/f40511bd2dc24376ad090c47a8d5f353 2024-11-07T09:36:09,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/9973be5373334892a5704b8f0a63f8b4 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/9973be5373334892a5704b8f0a63f8b4 2024-11-07T09:36:09,672 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/9973be5373334892a5704b8f0a63f8b4, entries=200, sequenceid=280, filesize=14.4 K 2024-11-07T09:36:09,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/dffd9b43a7cd49f0936e35152fa012dd as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/dffd9b43a7cd49f0936e35152fa012dd 2024-11-07T09:36:09,679 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/dffd9b43a7cd49f0936e35152fa012dd, entries=150, sequenceid=280, filesize=12.0 K 2024-11-07T09:36:09,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/f40511bd2dc24376ad090c47a8d5f353 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/f40511bd2dc24376ad090c47a8d5f353 2024-11-07T09:36:09,700 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/f40511bd2dc24376ad090c47a8d5f353, entries=150, sequenceid=280, filesize=12.0 K 2024-11-07T09:36:09,703 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 116dccc5098610cd38ab507cb3daad26 in 526ms, sequenceid=280, compaction requested=false 2024-11-07T09:36:09,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:09,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-07T09:36:09,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-07T09:36:09,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:09,803 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 116dccc5098610cd38ab507cb3daad26 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-07T09:36:09,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/2adec459751c44a690d4d0db75729f69 is 50, key is test_row_0/A:col10/1730972169199/Put/seqid=0 2024-11-07T09:36:09,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741894_1070 (size=12301) 2024-11-07T09:36:09,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 116dccc5098610cd38ab507cb3daad26 2024-11-07T09:36:09,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. as already flushing 2024-11-07T09:36:09,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:09,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972229876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:09,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:09,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:09,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:09,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:09,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972229882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:09,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:09,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:09,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:09,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:09,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972229883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:09,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:09,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:09,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:09,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-07T09:36:09,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:09,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972229992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:09,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:09,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:09,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:09,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:09,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:09,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972229993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:09,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972229993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:09,997 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:09,997 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:09,997 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:09,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:09,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:09,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:10,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:10,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972230203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:10,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:10,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:10,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:10,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:10,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972230204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:10,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:10,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:10,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:10,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:10,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972230204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:10,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:10,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:10,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:10,214 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/2adec459751c44a690d4d0db75729f69 2024-11-07T09:36:10,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/310be4268321483caee35b189eb83a12 is 50, key is test_row_0/B:col10/1730972169199/Put/seqid=0 2024-11-07T09:36:10,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741895_1071 (size=12301) 2024-11-07T09:36:10,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-07T09:36:10,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:10,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972230513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:10,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:10,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:10,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:10,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972230513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:10,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:10,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:10,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:10,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:10,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:10,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972230514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:10,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:10,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:10,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:10,629 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/310be4268321483caee35b189eb83a12 2024-11-07T09:36:10,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/0b307e79369644f3ad116943bb008516 is 50, key is test_row_0/C:col10/1730972169199/Put/seqid=0 2024-11-07T09:36:10,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741896_1072 (size=12301) 2024-11-07T09:36:10,650 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/0b307e79369644f3ad116943bb008516 2024-11-07T09:36:10,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/2adec459751c44a690d4d0db75729f69 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/2adec459751c44a690d4d0db75729f69 2024-11-07T09:36:10,663 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/2adec459751c44a690d4d0db75729f69, entries=150, sequenceid=294, filesize=12.0 K 2024-11-07T09:36:10,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/310be4268321483caee35b189eb83a12 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/310be4268321483caee35b189eb83a12 2024-11-07T09:36:10,670 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/310be4268321483caee35b189eb83a12, entries=150, sequenceid=294, filesize=12.0 K 2024-11-07T09:36:10,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/0b307e79369644f3ad116943bb008516 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/0b307e79369644f3ad116943bb008516 2024-11-07T09:36:10,676 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/0b307e79369644f3ad116943bb008516, entries=150, sequenceid=294, filesize=12.0 K 2024-11-07T09:36:10,677 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 116dccc5098610cd38ab507cb3daad26 in 874ms, sequenceid=294, compaction requested=true 2024-11-07T09:36:10,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:10,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:10,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-07T09:36:10,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-07T09:36:10,682 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-07T09:36:10,682 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0310 sec 2024-11-07T09:36:10,685 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.0390 sec 2024-11-07T09:36:10,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-07T09:36:10,783 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:36:10,784 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T09:36:10,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-07T09:36:10,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-07T09:36:10,790 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T09:36:10,791 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T09:36:10,791 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T09:36:10,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-07T09:36:10,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-07T09:36:10,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:10,953 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 116dccc5098610cd38ab507cb3daad26 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-07T09:36:10,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/bd6abe3fc8ca4682a850015d73b117a9 is 50, key is test_row_0/A:col10/1730972169870/Put/seqid=0 2024-11-07T09:36:10,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741897_1073 (size=12301) 2024-11-07T09:36:11,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 116dccc5098610cd38ab507cb3daad26 2024-11-07T09:36:11,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. as already flushing 2024-11-07T09:36:11,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:11,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972231037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:11,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:11,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:11,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:11,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:11,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972231039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:11,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:11,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:11,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:11,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:11,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972231040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:11,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:11,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:11,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:11,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-07T09:36:11,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:11,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972231153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:11,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:11,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:11,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:11,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972231154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:11,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:11,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:11,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:11,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:11,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972231154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:11,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:11,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:11,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:11,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:11,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:11,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972231363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:11,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:11,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:11,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:11,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:11,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972231364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:11,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:11,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:11,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:11,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:11,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972231364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:11,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:11,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:11,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:11,377 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/bd6abe3fc8ca4682a850015d73b117a9 2024-11-07T09:36:11,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/55618b88d39043f3b9a6fe65b34e9670 is 50, key is test_row_0/B:col10/1730972169870/Put/seqid=0 2024-11-07T09:36:11,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741898_1074 (size=12301) 2024-11-07T09:36:11,398 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/55618b88d39043f3b9a6fe65b34e9670 2024-11-07T09:36:11,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/ae192a51f92942568245bf9ab180c8ff is 50, key is test_row_0/C:col10/1730972169870/Put/seqid=0 2024-11-07T09:36:11,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741899_1075 (size=12301) 2024-11-07T09:36:11,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-07T09:36:11,414 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/ae192a51f92942568245bf9ab180c8ff 2024-11-07T09:36:11,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/bd6abe3fc8ca4682a850015d73b117a9 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/bd6abe3fc8ca4682a850015d73b117a9 2024-11-07T09:36:11,429 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/bd6abe3fc8ca4682a850015d73b117a9, entries=150, sequenceid=317, filesize=12.0 K 2024-11-07T09:36:11,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/55618b88d39043f3b9a6fe65b34e9670 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/55618b88d39043f3b9a6fe65b34e9670 2024-11-07T09:36:11,435 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/55618b88d39043f3b9a6fe65b34e9670, entries=150, sequenceid=317, filesize=12.0 K 2024-11-07T09:36:11,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/ae192a51f92942568245bf9ab180c8ff as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/ae192a51f92942568245bf9ab180c8ff 2024-11-07T09:36:11,443 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/ae192a51f92942568245bf9ab180c8ff, entries=150, sequenceid=317, filesize=12.0 K 2024-11-07T09:36:11,444 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 116dccc5098610cd38ab507cb3daad26 in 491ms, sequenceid=317, compaction requested=true 2024-11-07T09:36:11,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:11,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:11,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-07T09:36:11,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-07T09:36:11,449 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-07T09:36:11,449 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 655 msec 2024-11-07T09:36:11,452 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 667 msec 2024-11-07T09:36:11,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 116dccc5098610cd38ab507cb3daad26 2024-11-07T09:36:11,677 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:36:11,677 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 116dccc5098610cd38ab507cb3daad26 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-07T09:36:11,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/b6e017b923dc478a92b2daed0c2a58c3 is 50, key is test_row_0/A:col10/1730972171676/Put/seqid=0 2024-11-07T09:36:11,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741900_1076 (size=14741) 2024-11-07T09:36:11,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:11,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972231710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:11,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:11,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:11,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:11,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972231711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:11,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:11,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:11,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:11,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:11,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:11,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972231714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:11,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:11,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:11,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:11,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:11,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972231823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:11,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:11,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:11,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972231823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:11,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:11,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:11,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:11,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:11,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:11,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:11,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972231823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:11,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:11,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:11,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:11,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-07T09:36:11,924 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:36:11,925 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T09:36:11,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-07T09:36:11,928 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T09:36:11,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-07T09:36:11,929 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T09:36:11,929 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T09:36:12,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-07T09:36:12,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:12,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972232035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:12,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:12,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:12,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972232035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:12,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:12,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:12,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:12,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:12,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:12,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:12,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972232038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:12,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:12,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:12,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:12,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-07T09:36:12,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:12,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. as already flushing 2024-11-07T09:36:12,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:12,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:12,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:12,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:12,100 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/b6e017b923dc478a92b2daed0c2a58c3 2024-11-07T09:36:12,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/00aae78f067a4dc2813d7e7c256355fa is 50, key is test_row_0/B:col10/1730972171676/Put/seqid=0 2024-11-07T09:36:12,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741901_1077 (size=12301) 2024-11-07T09:36:12,128 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/00aae78f067a4dc2813d7e7c256355fa 2024-11-07T09:36:12,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/0a324d7036dd4ed0851570f49ef5505d is 50, key is test_row_0/C:col10/1730972171676/Put/seqid=0 2024-11-07T09:36:12,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741902_1078 (size=12301) 2024-11-07T09:36:12,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-07T09:36:12,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:12,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. as already flushing 2024-11-07T09:36:12,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:12,238 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:12,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:12,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:12,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-07T09:36:12,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:12,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972232354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:12,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:12,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:12,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:12,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:12,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972232354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:12,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:12,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972232354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:12,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:12,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:12,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:12,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:12,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:12,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:12,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-07T09:36:12,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:12,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. as already flushing 2024-11-07T09:36:12,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:12,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:12,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:12,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:12,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-07T09:36:12,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:12,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. as already flushing 2024-11-07T09:36:12,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:12,547 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:12,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:12,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:12,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-07T09:36:12,557 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/0a324d7036dd4ed0851570f49ef5505d 2024-11-07T09:36:12,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/b6e017b923dc478a92b2daed0c2a58c3 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/b6e017b923dc478a92b2daed0c2a58c3 2024-11-07T09:36:12,580 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/b6e017b923dc478a92b2daed0c2a58c3, entries=200, sequenceid=332, filesize=14.4 K 2024-11-07T09:36:12,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/00aae78f067a4dc2813d7e7c256355fa as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/00aae78f067a4dc2813d7e7c256355fa 2024-11-07T09:36:12,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/00aae78f067a4dc2813d7e7c256355fa, entries=150, sequenceid=332, filesize=12.0 K 2024-11-07T09:36:12,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/0a324d7036dd4ed0851570f49ef5505d as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/0a324d7036dd4ed0851570f49ef5505d 2024-11-07T09:36:12,596 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/0a324d7036dd4ed0851570f49ef5505d, entries=150, sequenceid=332, filesize=12.0 K 2024-11-07T09:36:12,599 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 116dccc5098610cd38ab507cb3daad26 in 921ms, sequenceid=332, compaction requested=true 2024-11-07T09:36:12,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:12,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 116dccc5098610cd38ab507cb3daad26:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T09:36:12,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:12,599 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-07T09:36:12,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 116dccc5098610cd38ab507cb3daad26:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T09:36:12,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:12,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 116dccc5098610cd38ab507cb3daad26:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:36:12,599 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-07T09:36:12,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:36:12,602 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61935 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-07T09:36:12,602 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 66815 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-07T09:36:12,602 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 116dccc5098610cd38ab507cb3daad26/A is initiating minor compaction (all files) 2024-11-07T09:36:12,602 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 116dccc5098610cd38ab507cb3daad26/B is initiating minor compaction (all files) 2024-11-07T09:36:12,603 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 116dccc5098610cd38ab507cb3daad26/A in TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:12,603 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/c26bf73380a5467b9d5ac64f99b71463, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/9973be5373334892a5704b8f0a63f8b4, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/2adec459751c44a690d4d0db75729f69, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/bd6abe3fc8ca4682a850015d73b117a9, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/b6e017b923dc478a92b2daed0c2a58c3] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp, totalSize=65.2 K 2024-11-07T09:36:12,603 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 116dccc5098610cd38ab507cb3daad26/B in TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:12,603 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/6951fbd9c41c4a70b77f0c14961dd8bc, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/dffd9b43a7cd49f0936e35152fa012dd, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/310be4268321483caee35b189eb83a12, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/55618b88d39043f3b9a6fe65b34e9670, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/00aae78f067a4dc2813d7e7c256355fa] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp, totalSize=60.5 K 2024-11-07T09:36:12,604 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting c26bf73380a5467b9d5ac64f99b71463, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1730972166832 2024-11-07T09:36:12,604 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 6951fbd9c41c4a70b77f0c14961dd8bc, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1730972166832 2024-11-07T09:36:12,605 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9973be5373334892a5704b8f0a63f8b4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1730972168020 2024-11-07T09:36:12,605 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting dffd9b43a7cd49f0936e35152fa012dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1730972168026 2024-11-07T09:36:12,605 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2adec459751c44a690d4d0db75729f69, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1730972169198 2024-11-07T09:36:12,606 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 310be4268321483caee35b189eb83a12, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1730972169198 2024-11-07T09:36:12,606 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd6abe3fc8ca4682a850015d73b117a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1730972169870 2024-11-07T09:36:12,606 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 55618b88d39043f3b9a6fe65b34e9670, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1730972169870 2024-11-07T09:36:12,607 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6e017b923dc478a92b2daed0c2a58c3, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1730972171031 2024-11-07T09:36:12,607 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 00aae78f067a4dc2813d7e7c256355fa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1730972171039 2024-11-07T09:36:12,632 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 116dccc5098610cd38ab507cb3daad26#B#compaction#66 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:12,633 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/38afe9c89e6c42a88cb0925de7d21425 is 50, key is test_row_0/B:col10/1730972171676/Put/seqid=0 2024-11-07T09:36:12,637 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 116dccc5098610cd38ab507cb3daad26#A#compaction#67 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:12,638 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/821882ae46ef4a4bb4e7f3e450ffe4cc is 50, key is test_row_0/A:col10/1730972171676/Put/seqid=0 2024-11-07T09:36:12,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741904_1080 (size=13051) 2024-11-07T09:36:12,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741903_1079 (size=13051) 2024-11-07T09:36:12,657 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/821882ae46ef4a4bb4e7f3e450ffe4cc as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/821882ae46ef4a4bb4e7f3e450ffe4cc 2024-11-07T09:36:12,660 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/38afe9c89e6c42a88cb0925de7d21425 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/38afe9c89e6c42a88cb0925de7d21425 2024-11-07T09:36:12,669 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 116dccc5098610cd38ab507cb3daad26/A of 116dccc5098610cd38ab507cb3daad26 into 821882ae46ef4a4bb4e7f3e450ffe4cc(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:12,669 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:12,669 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., storeName=116dccc5098610cd38ab507cb3daad26/A, priority=11, startTime=1730972172599; duration=0sec 2024-11-07T09:36:12,670 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:36:12,670 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 116dccc5098610cd38ab507cb3daad26:A 2024-11-07T09:36:12,670 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-07T09:36:12,670 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 116dccc5098610cd38ab507cb3daad26/B of 116dccc5098610cd38ab507cb3daad26 into 38afe9c89e6c42a88cb0925de7d21425(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:12,670 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:12,672 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., storeName=116dccc5098610cd38ab507cb3daad26/B, priority=11, startTime=1730972172599; duration=0sec 2024-11-07T09:36:12,672 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:12,672 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 116dccc5098610cd38ab507cb3daad26:B 2024-11-07T09:36:12,673 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61935 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-07T09:36:12,673 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 116dccc5098610cd38ab507cb3daad26/C is initiating minor compaction (all files) 2024-11-07T09:36:12,673 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 116dccc5098610cd38ab507cb3daad26/C in TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:12,673 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/970c7ecb9b64484585c148efd66bb851, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/f40511bd2dc24376ad090c47a8d5f353, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/0b307e79369644f3ad116943bb008516, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/ae192a51f92942568245bf9ab180c8ff, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/0a324d7036dd4ed0851570f49ef5505d] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp, totalSize=60.5 K 2024-11-07T09:36:12,674 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 970c7ecb9b64484585c148efd66bb851, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1730972166832 2024-11-07T09:36:12,675 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting f40511bd2dc24376ad090c47a8d5f353, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1730972168026 2024-11-07T09:36:12,675 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b307e79369644f3ad116943bb008516, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1730972169198 2024-11-07T09:36:12,675 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae192a51f92942568245bf9ab180c8ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1730972169870 2024-11-07T09:36:12,676 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a324d7036dd4ed0851570f49ef5505d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1730972171039 2024-11-07T09:36:12,694 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 116dccc5098610cd38ab507cb3daad26#C#compaction#68 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:12,695 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/8a254dfe0e624ca3b2c757278daee4fd is 50, key is test_row_0/C:col10/1730972171676/Put/seqid=0 2024-11-07T09:36:12,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-07T09:36:12,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:12,703 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 116dccc5098610cd38ab507cb3daad26 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-07T09:36:12,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/38c0949cdd214f83b07167e472905239 is 50, key is test_row_0/A:col10/1730972171711/Put/seqid=0 2024-11-07T09:36:12,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741905_1081 (size=13051) 2024-11-07T09:36:12,728 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/8a254dfe0e624ca3b2c757278daee4fd as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/8a254dfe0e624ca3b2c757278daee4fd 2024-11-07T09:36:12,736 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 116dccc5098610cd38ab507cb3daad26/C of 116dccc5098610cd38ab507cb3daad26 into 8a254dfe0e624ca3b2c757278daee4fd(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:12,736 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:12,736 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., storeName=116dccc5098610cd38ab507cb3daad26/C, priority=11, startTime=1730972172599; duration=0sec 2024-11-07T09:36:12,736 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:12,736 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 116dccc5098610cd38ab507cb3daad26:C 2024-11-07T09:36:12,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741906_1082 (size=12301) 2024-11-07T09:36:12,747 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/38c0949cdd214f83b07167e472905239 2024-11-07T09:36:12,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/be320cbd5ad64be4b404667c8746bbd5 is 50, key is test_row_0/B:col10/1730972171711/Put/seqid=0 2024-11-07T09:36:12,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741907_1083 (size=12301) 2024-11-07T09:36:12,792 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/be320cbd5ad64be4b404667c8746bbd5 2024-11-07T09:36:12,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/f3337699030041d8a1f5d402e35a7506 is 50, key is test_row_0/C:col10/1730972171711/Put/seqid=0 2024-11-07T09:36:12,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741908_1084 (size=12301) 2024-11-07T09:36:12,811 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/f3337699030041d8a1f5d402e35a7506 2024-11-07T09:36:12,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/38c0949cdd214f83b07167e472905239 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/38c0949cdd214f83b07167e472905239 2024-11-07T09:36:12,828 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/38c0949cdd214f83b07167e472905239, entries=150, sequenceid=355, filesize=12.0 K 2024-11-07T09:36:12,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/be320cbd5ad64be4b404667c8746bbd5 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/be320cbd5ad64be4b404667c8746bbd5 2024-11-07T09:36:12,839 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/be320cbd5ad64be4b404667c8746bbd5, entries=150, sequenceid=355, filesize=12.0 K 2024-11-07T09:36:12,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/f3337699030041d8a1f5d402e35a7506 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/f3337699030041d8a1f5d402e35a7506 2024-11-07T09:36:12,849 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/f3337699030041d8a1f5d402e35a7506, entries=150, sequenceid=355, filesize=12.0 K 2024-11-07T09:36:12,851 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=0 B/0 for 116dccc5098610cd38ab507cb3daad26 in 149ms, sequenceid=355, compaction requested=false 2024-11-07T09:36:12,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:12,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:12,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-07T09:36:12,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-07T09:36:12,857 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-07T09:36:12,857 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 924 msec 2024-11-07T09:36:12,860 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 933 msec 2024-11-07T09:36:12,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 116dccc5098610cd38ab507cb3daad26 2024-11-07T09:36:12,894 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:36:12,895 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 116dccc5098610cd38ab507cb3daad26 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T09:36:12,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/1ff56655b1b544fb959350edef75f74c is 50, key is test_row_0/A:col10/1730972172893/Put/seqid=0 2024-11-07T09:36:12,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741909_1085 (size=12301) 2024-11-07T09:36:12,926 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/1ff56655b1b544fb959350edef75f74c 2024-11-07T09:36:12,941 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/35b6ff9e469547a1a56be6d37d956276 is 50, key is test_row_0/B:col10/1730972172893/Put/seqid=0 2024-11-07T09:36:12,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741910_1086 (size=12301) 2024-11-07T09:36:12,960 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/35b6ff9e469547a1a56be6d37d956276 2024-11-07T09:36:12,976 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/b0d44551c26d4a478aef1d6ee446b8a3 is 50, key is test_row_0/C:col10/1730972172893/Put/seqid=0 2024-11-07T09:36:12,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:12,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972232971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:12,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:12,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:12,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:12,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:12,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972232972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:12,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:12,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:12,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:12,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:12,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972232974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:12,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:12,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:12,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:12,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741911_1087 (size=12301) 2024-11-07T09:36:13,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-07T09:36:13,064 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:36:13,064 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T09:36:13,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-07T09:36:13,068 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T09:36:13,069 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T09:36:13,069 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T09:36:13,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-07T09:36:13,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:13,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972233083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:13,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:13,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:13,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972233083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:13,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:13,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:13,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:13,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:13,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:13,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:13,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972233084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:13,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:13,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:13,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:13,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-07T09:36:13,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-07T09:36:13,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:13,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. as already flushing 2024-11-07T09:36:13,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:13,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:13,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:13,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:13,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:13,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972233293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:13,296 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:13,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:13,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:13,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:13,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972233293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:13,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:13,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972233293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:13,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:13,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:13,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:13,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:13,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:13,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:13,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-07T09:36:13,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:13,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. as already flushing 2024-11-07T09:36:13,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:13,377 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:13,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:13,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:13,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-07T09:36:13,394 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/b0d44551c26d4a478aef1d6ee446b8a3 2024-11-07T09:36:13,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/1ff56655b1b544fb959350edef75f74c as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/1ff56655b1b544fb959350edef75f74c 2024-11-07T09:36:13,406 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/1ff56655b1b544fb959350edef75f74c, entries=150, sequenceid=367, filesize=12.0 K 2024-11-07T09:36:13,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/35b6ff9e469547a1a56be6d37d956276 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/35b6ff9e469547a1a56be6d37d956276 2024-11-07T09:36:13,412 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/35b6ff9e469547a1a56be6d37d956276, entries=150, sequenceid=367, filesize=12.0 K 2024-11-07T09:36:13,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/b0d44551c26d4a478aef1d6ee446b8a3 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/b0d44551c26d4a478aef1d6ee446b8a3 2024-11-07T09:36:13,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/b0d44551c26d4a478aef1d6ee446b8a3, entries=150, sequenceid=367, filesize=12.0 K 2024-11-07T09:36:13,419 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 116dccc5098610cd38ab507cb3daad26 in 523ms, sequenceid=367, compaction requested=true 2024-11-07T09:36:13,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:13,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 116dccc5098610cd38ab507cb3daad26:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T09:36:13,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:13,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 116dccc5098610cd38ab507cb3daad26:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T09:36:13,419 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:36:13,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:13,419 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:36:13,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 116dccc5098610cd38ab507cb3daad26:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:36:13,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:36:13,420 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:36:13,420 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:36:13,420 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 116dccc5098610cd38ab507cb3daad26/A is initiating minor compaction (all files) 2024-11-07T09:36:13,420 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 116dccc5098610cd38ab507cb3daad26/B is initiating minor compaction (all files) 2024-11-07T09:36:13,420 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 116dccc5098610cd38ab507cb3daad26/A in TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:13,421 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 116dccc5098610cd38ab507cb3daad26/B in TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:13,421 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/821882ae46ef4a4bb4e7f3e450ffe4cc, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/38c0949cdd214f83b07167e472905239, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/1ff56655b1b544fb959350edef75f74c] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp, totalSize=36.8 K 2024-11-07T09:36:13,421 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/38afe9c89e6c42a88cb0925de7d21425, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/be320cbd5ad64be4b404667c8746bbd5, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/35b6ff9e469547a1a56be6d37d956276] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp, totalSize=36.8 K 2024-11-07T09:36:13,421 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 821882ae46ef4a4bb4e7f3e450ffe4cc, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1730972171039 2024-11-07T09:36:13,421 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 38afe9c89e6c42a88cb0925de7d21425, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1730972171039 2024-11-07T09:36:13,422 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38c0949cdd214f83b07167e472905239, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1730972171701 2024-11-07T09:36:13,422 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting be320cbd5ad64be4b404667c8746bbd5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1730972171701 2024-11-07T09:36:13,422 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ff56655b1b544fb959350edef75f74c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1730972172876 2024-11-07T09:36:13,422 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 35b6ff9e469547a1a56be6d37d956276, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1730972172876 2024-11-07T09:36:13,440 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 116dccc5098610cd38ab507cb3daad26#A#compaction#75 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:13,441 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/e2ed47e7eab343f497619c1f23cff3cf is 50, key is test_row_0/A:col10/1730972172893/Put/seqid=0 2024-11-07T09:36:13,451 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 116dccc5098610cd38ab507cb3daad26#B#compaction#76 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:13,451 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/544f7a4e9dbc4f0b9a7bb2d93500253a is 50, key is test_row_0/B:col10/1730972172893/Put/seqid=0 2024-11-07T09:36:13,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741912_1088 (size=13153) 2024-11-07T09:36:13,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741913_1089 (size=13153) 2024-11-07T09:36:13,487 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/e2ed47e7eab343f497619c1f23cff3cf as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/e2ed47e7eab343f497619c1f23cff3cf 2024-11-07T09:36:13,492 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/544f7a4e9dbc4f0b9a7bb2d93500253a as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/544f7a4e9dbc4f0b9a7bb2d93500253a 2024-11-07T09:36:13,498 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 116dccc5098610cd38ab507cb3daad26/A of 116dccc5098610cd38ab507cb3daad26 into e2ed47e7eab343f497619c1f23cff3cf(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:13,498 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:13,498 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., storeName=116dccc5098610cd38ab507cb3daad26/A, priority=13, startTime=1730972173419; duration=0sec 2024-11-07T09:36:13,499 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:36:13,499 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 116dccc5098610cd38ab507cb3daad26:A 2024-11-07T09:36:13,499 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:36:13,501 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 116dccc5098610cd38ab507cb3daad26/B of 116dccc5098610cd38ab507cb3daad26 into 544f7a4e9dbc4f0b9a7bb2d93500253a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:13,501 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:13,501 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., storeName=116dccc5098610cd38ab507cb3daad26/B, priority=13, startTime=1730972173419; duration=0sec 2024-11-07T09:36:13,501 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:13,501 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 116dccc5098610cd38ab507cb3daad26:B 2024-11-07T09:36:13,502 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:36:13,502 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 116dccc5098610cd38ab507cb3daad26/C is initiating minor compaction (all files) 2024-11-07T09:36:13,502 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 116dccc5098610cd38ab507cb3daad26/C in TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:13,503 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/8a254dfe0e624ca3b2c757278daee4fd, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/f3337699030041d8a1f5d402e35a7506, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/b0d44551c26d4a478aef1d6ee446b8a3] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp, totalSize=36.8 K 2024-11-07T09:36:13,504 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a254dfe0e624ca3b2c757278daee4fd, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1730972171039 2024-11-07T09:36:13,504 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3337699030041d8a1f5d402e35a7506, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1730972171701 2024-11-07T09:36:13,505 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0d44551c26d4a478aef1d6ee446b8a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1730972172876 2024-11-07T09:36:13,517 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 116dccc5098610cd38ab507cb3daad26#C#compaction#77 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:13,518 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/17ec7b8ae36e462184a008c0637152cd is 50, key is test_row_0/C:col10/1730972172893/Put/seqid=0 2024-11-07T09:36:13,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-07T09:36:13,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:13,531 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 116dccc5098610cd38ab507cb3daad26 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-07T09:36:13,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741914_1090 (size=13153) 2024-11-07T09:36:13,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/0ee80eee76e644c288f3f4a350e36354 is 50, key is test_row_0/A:col10/1730972172971/Put/seqid=0 2024-11-07T09:36:13,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741915_1091 (size=12301) 2024-11-07T09:36:13,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 116dccc5098610cd38ab507cb3daad26 2024-11-07T09:36:13,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. as already flushing 2024-11-07T09:36:13,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:13,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972233613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:13,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:13,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972233615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:13,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:13,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:13,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:13,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:13,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:13,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:13,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:13,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972233618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:13,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:13,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:13,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:13,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:13,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1730972233643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:13,648 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 9f1eb259a29b,42895,1730972153970 for 'test_row_1' in 116dccc5098610cd38ab507cb3daad26 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T09:36:13,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:13,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:13,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:13,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:13,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33506 deadline: 1730972233663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:13,664 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 9f1eb259a29b,42895,1730972153970 for 'test_row_2' in 116dccc5098610cd38ab507cb3daad26 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8232 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T09:36:13,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:13,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:13,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:13,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-07T09:36:13,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:13,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972233723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:13,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:13,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:13,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:13,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972233723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:13,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:13,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:13,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:13,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:13,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:13,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972233733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:13,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:13,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:13,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:13,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:13,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972233934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:13,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:13,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:13,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:13,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:13,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972233934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:13,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:13,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:13,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:13,940 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/17ec7b8ae36e462184a008c0637152cd as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/17ec7b8ae36e462184a008c0637152cd 2024-11-07T09:36:13,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:13,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972233943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:13,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:13,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:13,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:13,947 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 116dccc5098610cd38ab507cb3daad26/C of 116dccc5098610cd38ab507cb3daad26 into 17ec7b8ae36e462184a008c0637152cd(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:13,947 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:13,947 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., storeName=116dccc5098610cd38ab507cb3daad26/C, priority=13, startTime=1730972173419; duration=0sec 2024-11-07T09:36:13,947 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:13,947 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 116dccc5098610cd38ab507cb3daad26:C 2024-11-07T09:36:13,973 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/0ee80eee76e644c288f3f4a350e36354 2024-11-07T09:36:13,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/6519dcaa9ec44f43a9ec9733c510cafc is 50, key is test_row_0/B:col10/1730972172971/Put/seqid=0 2024-11-07T09:36:13,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741916_1092 (size=12301) 2024-11-07T09:36:14,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-07T09:36:14,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:14,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972234242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:14,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:14,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:14,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:14,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:14,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972234243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:14,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:14,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:14,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:14,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:14,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972234253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:14,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:14,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:14,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:14,391 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/6519dcaa9ec44f43a9ec9733c510cafc 2024-11-07T09:36:14,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/81a1a37cb4f941d881a4539ad3fb96eb is 50, key is test_row_0/C:col10/1730972172971/Put/seqid=0 2024-11-07T09:36:14,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741917_1093 (size=12301) 2024-11-07T09:36:14,406 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/81a1a37cb4f941d881a4539ad3fb96eb 2024-11-07T09:36:14,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/0ee80eee76e644c288f3f4a350e36354 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/0ee80eee76e644c288f3f4a350e36354 2024-11-07T09:36:14,420 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/0ee80eee76e644c288f3f4a350e36354, entries=150, sequenceid=395, filesize=12.0 K 2024-11-07T09:36:14,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/6519dcaa9ec44f43a9ec9733c510cafc as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/6519dcaa9ec44f43a9ec9733c510cafc 2024-11-07T09:36:14,427 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/6519dcaa9ec44f43a9ec9733c510cafc, entries=150, sequenceid=395, filesize=12.0 K 2024-11-07T09:36:14,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/81a1a37cb4f941d881a4539ad3fb96eb as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/81a1a37cb4f941d881a4539ad3fb96eb 2024-11-07T09:36:14,437 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/81a1a37cb4f941d881a4539ad3fb96eb, entries=150, sequenceid=395, filesize=12.0 K 2024-11-07T09:36:14,438 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 116dccc5098610cd38ab507cb3daad26 in 907ms, sequenceid=395, compaction requested=false 2024-11-07T09:36:14,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:14,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:14,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-07T09:36:14,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-07T09:36:14,446 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-07T09:36:14,446 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3730 sec 2024-11-07T09:36:14,449 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.3830 sec 2024-11-07T09:36:14,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 116dccc5098610cd38ab507cb3daad26 2024-11-07T09:36:14,755 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:36:14,755 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 116dccc5098610cd38ab507cb3daad26 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T09:36:14,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/a35fd9ed4dfe45bf997f43d535c79f20 is 50, key is test_row_0/A:col10/1730972173611/Put/seqid=0 2024-11-07T09:36:14,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741918_1094 (size=12301) 2024-11-07T09:36:14,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:14,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972234798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:14,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:14,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:14,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:14,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:14,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972234799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:14,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:14,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:14,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:14,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:14,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972234801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:14,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:14,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:14,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:14,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:14,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972234903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:14,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:14,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:14,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:14,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:14,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972234913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:14,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:14,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:14,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:14,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:14,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972234913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:14,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:14,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:14,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:15,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:15,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972235113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:15,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:15,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:15,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:15,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:15,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972235122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:15,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:15,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:15,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:15,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972235122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:15,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:15,128 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:15,128 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:15,128 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:15,176 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/a35fd9ed4dfe45bf997f43d535c79f20 2024-11-07T09:36:15,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/5cb23d5a5d3b4fc8952baaeb636938eb is 50, key is test_row_0/B:col10/1730972173611/Put/seqid=0 2024-11-07T09:36:15,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741919_1095 (size=12301) 2024-11-07T09:36:15,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-07T09:36:15,214 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:36:15,214 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T09:36:15,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-07T09:36:15,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-07T09:36:15,216 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T09:36:15,217 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T09:36:15,217 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T09:36:15,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-07T09:36:15,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-07T09:36:15,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:15,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. as already flushing 2024-11-07T09:36:15,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:15,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:15,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:15,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:15,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:15,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972235423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:15,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:15,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:15,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:15,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:15,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972235433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:15,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:15,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:15,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:15,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:15,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972235433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:15,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:15,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:15,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:15,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-07T09:36:15,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:15,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. as already flushing 2024-11-07T09:36:15,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:15,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:15,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:15,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:15,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-07T09:36:15,591 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/5cb23d5a5d3b4fc8952baaeb636938eb 2024-11-07T09:36:15,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/51a9242399aa42d0b82b5179ba0eb3c5 is 50, key is test_row_0/C:col10/1730972173611/Put/seqid=0 2024-11-07T09:36:15,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741920_1096 (size=12301) 2024-11-07T09:36:15,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-07T09:36:15,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:15,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. as already flushing 2024-11-07T09:36:15,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:15,677 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:15,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:15,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:15,830 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-07T09:36:15,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:15,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. as already flushing 2024-11-07T09:36:15,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:15,831 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:15,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:15,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:15,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-07T09:36:15,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:15,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972235933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:15,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:15,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:15,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:15,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:15,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972235942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:15,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:15,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:15,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:15,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:15,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972235943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:15,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:15,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:15,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:15,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-07T09:36:15,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:15,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. as already flushing 2024-11-07T09:36:15,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:15,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:15,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:15,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:16,007 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/51a9242399aa42d0b82b5179ba0eb3c5 2024-11-07T09:36:16,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/a35fd9ed4dfe45bf997f43d535c79f20 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/a35fd9ed4dfe45bf997f43d535c79f20 2024-11-07T09:36:16,018 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/a35fd9ed4dfe45bf997f43d535c79f20, entries=150, sequenceid=407, filesize=12.0 K 2024-11-07T09:36:16,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/5cb23d5a5d3b4fc8952baaeb636938eb as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/5cb23d5a5d3b4fc8952baaeb636938eb 2024-11-07T09:36:16,025 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/5cb23d5a5d3b4fc8952baaeb636938eb, entries=150, sequenceid=407, filesize=12.0 K 2024-11-07T09:36:16,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/51a9242399aa42d0b82b5179ba0eb3c5 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/51a9242399aa42d0b82b5179ba0eb3c5 2024-11-07T09:36:16,034 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/51a9242399aa42d0b82b5179ba0eb3c5, entries=150, sequenceid=407, filesize=12.0 K 2024-11-07T09:36:16,036 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 116dccc5098610cd38ab507cb3daad26 in 1281ms, sequenceid=407, compaction requested=true 2024-11-07T09:36:16,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:16,036 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:36:16,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 116dccc5098610cd38ab507cb3daad26:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T09:36:16,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:16,038 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:36:16,038 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:36:16,038 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 116dccc5098610cd38ab507cb3daad26/A is initiating minor compaction (all files) 2024-11-07T09:36:16,038 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 116dccc5098610cd38ab507cb3daad26/A in TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:16,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 116dccc5098610cd38ab507cb3daad26:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T09:36:16,038 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/e2ed47e7eab343f497619c1f23cff3cf, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/0ee80eee76e644c288f3f4a350e36354, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/a35fd9ed4dfe45bf997f43d535c79f20] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp, totalSize=36.9 K 2024-11-07T09:36:16,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:16,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 116dccc5098610cd38ab507cb3daad26:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:36:16,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:36:16,039 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2ed47e7eab343f497619c1f23cff3cf, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1730972172876 2024-11-07T09:36:16,039 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ee80eee76e644c288f3f4a350e36354, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1730972172924 2024-11-07T09:36:16,040 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:36:16,040 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting a35fd9ed4dfe45bf997f43d535c79f20, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1730972173611 2024-11-07T09:36:16,040 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 116dccc5098610cd38ab507cb3daad26/B is initiating minor compaction (all files) 2024-11-07T09:36:16,040 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 116dccc5098610cd38ab507cb3daad26/B in TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:16,040 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/544f7a4e9dbc4f0b9a7bb2d93500253a, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/6519dcaa9ec44f43a9ec9733c510cafc, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/5cb23d5a5d3b4fc8952baaeb636938eb] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp, totalSize=36.9 K 2024-11-07T09:36:16,041 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 544f7a4e9dbc4f0b9a7bb2d93500253a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1730972172876 2024-11-07T09:36:16,041 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 6519dcaa9ec44f43a9ec9733c510cafc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1730972172924 2024-11-07T09:36:16,042 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cb23d5a5d3b4fc8952baaeb636938eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1730972173611 2024-11-07T09:36:16,051 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 116dccc5098610cd38ab507cb3daad26#A#compaction#84 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:16,052 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/5081740c127642c8ac59ea97d09508a0 is 50, key is test_row_0/A:col10/1730972173611/Put/seqid=0 2024-11-07T09:36:16,058 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 116dccc5098610cd38ab507cb3daad26#B#compaction#85 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:16,059 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/ff24f01394d34b8d90b1c4126c793358 is 50, key is test_row_0/B:col10/1730972173611/Put/seqid=0 2024-11-07T09:36:16,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741921_1097 (size=13255) 2024-11-07T09:36:16,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741922_1098 (size=13255) 2024-11-07T09:36:16,073 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/5081740c127642c8ac59ea97d09508a0 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/5081740c127642c8ac59ea97d09508a0 2024-11-07T09:36:16,082 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/ff24f01394d34b8d90b1c4126c793358 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/ff24f01394d34b8d90b1c4126c793358 2024-11-07T09:36:16,082 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 116dccc5098610cd38ab507cb3daad26/A of 116dccc5098610cd38ab507cb3daad26 into 5081740c127642c8ac59ea97d09508a0(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:16,082 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:16,082 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., storeName=116dccc5098610cd38ab507cb3daad26/A, priority=13, startTime=1730972176036; duration=0sec 2024-11-07T09:36:16,084 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:36:16,084 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 116dccc5098610cd38ab507cb3daad26:A 2024-11-07T09:36:16,085 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:36:16,087 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:36:16,087 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 116dccc5098610cd38ab507cb3daad26/C is initiating minor compaction (all files) 2024-11-07T09:36:16,087 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 116dccc5098610cd38ab507cb3daad26/C in TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:16,087 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/17ec7b8ae36e462184a008c0637152cd, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/81a1a37cb4f941d881a4539ad3fb96eb, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/51a9242399aa42d0b82b5179ba0eb3c5] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp, totalSize=36.9 K 2024-11-07T09:36:16,088 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17ec7b8ae36e462184a008c0637152cd, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1730972172876 2024-11-07T09:36:16,089 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81a1a37cb4f941d881a4539ad3fb96eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1730972172924 2024-11-07T09:36:16,090 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51a9242399aa42d0b82b5179ba0eb3c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1730972173611 2024-11-07T09:36:16,091 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 116dccc5098610cd38ab507cb3daad26/B of 116dccc5098610cd38ab507cb3daad26 into ff24f01394d34b8d90b1c4126c793358(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:16,091 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:16,092 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., storeName=116dccc5098610cd38ab507cb3daad26/B, priority=13, startTime=1730972176038; duration=0sec 2024-11-07T09:36:16,092 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:16,092 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 116dccc5098610cd38ab507cb3daad26:B 2024-11-07T09:36:16,099 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 116dccc5098610cd38ab507cb3daad26#C#compaction#86 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:16,100 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/27b72e1e6ccc4ae2b3700ebdde578ea5 is 50, key is test_row_0/C:col10/1730972173611/Put/seqid=0 2024-11-07T09:36:16,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741923_1099 (size=13255) 2024-11-07T09:36:16,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-07T09:36:16,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:16,139 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 116dccc5098610cd38ab507cb3daad26 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T09:36:16,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/530961d9eb6e44bea45ef59833ce0166 is 50, key is test_row_0/A:col10/1730972174799/Put/seqid=0 2024-11-07T09:36:16,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741924_1100 (size=12301) 2024-11-07T09:36:16,152 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/530961d9eb6e44bea45ef59833ce0166 2024-11-07T09:36:16,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/068fd5c449dc4f70883e0bcbab28e6a7 is 50, key is test_row_0/B:col10/1730972174799/Put/seqid=0 2024-11-07T09:36:16,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741925_1101 (size=12301) 2024-11-07T09:36:16,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-07T09:36:16,513 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/27b72e1e6ccc4ae2b3700ebdde578ea5 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/27b72e1e6ccc4ae2b3700ebdde578ea5 2024-11-07T09:36:16,520 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 116dccc5098610cd38ab507cb3daad26/C of 116dccc5098610cd38ab507cb3daad26 into 27b72e1e6ccc4ae2b3700ebdde578ea5(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:16,520 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:16,520 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., storeName=116dccc5098610cd38ab507cb3daad26/C, priority=13, startTime=1730972176038; duration=0sec 2024-11-07T09:36:16,520 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:16,520 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 116dccc5098610cd38ab507cb3daad26:C 2024-11-07T09:36:16,569 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/068fd5c449dc4f70883e0bcbab28e6a7 2024-11-07T09:36:16,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/8d2099c6245741649815a0789c72d4dc is 50, key is test_row_0/C:col10/1730972174799/Put/seqid=0 2024-11-07T09:36:16,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741926_1102 (size=12301) 2024-11-07T09:36:16,594 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/8d2099c6245741649815a0789c72d4dc 2024-11-07T09:36:16,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/530961d9eb6e44bea45ef59833ce0166 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/530961d9eb6e44bea45ef59833ce0166 2024-11-07T09:36:16,608 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/530961d9eb6e44bea45ef59833ce0166, entries=150, sequenceid=434, filesize=12.0 K 2024-11-07T09:36:16,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/068fd5c449dc4f70883e0bcbab28e6a7 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/068fd5c449dc4f70883e0bcbab28e6a7 2024-11-07T09:36:16,616 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/068fd5c449dc4f70883e0bcbab28e6a7, entries=150, sequenceid=434, filesize=12.0 K 2024-11-07T09:36:16,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/8d2099c6245741649815a0789c72d4dc as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/8d2099c6245741649815a0789c72d4dc 2024-11-07T09:36:16,622 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/8d2099c6245741649815a0789c72d4dc, entries=150, sequenceid=434, filesize=12.0 K 2024-11-07T09:36:16,623 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for 116dccc5098610cd38ab507cb3daad26 in 485ms, sequenceid=434, compaction requested=false 2024-11-07T09:36:16,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:16,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:16,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-07T09:36:16,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-07T09:36:16,627 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-07T09:36:16,627 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4080 sec 2024-11-07T09:36:16,630 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.4140 sec 2024-11-07T09:36:16,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 116dccc5098610cd38ab507cb3daad26 2024-11-07T09:36:16,966 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:36:16,966 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 116dccc5098610cd38ab507cb3daad26 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T09:36:16,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/2d90565175e24ee69163e3dad96b830c is 50, key is test_row_0/A:col10/1730972176965/Put/seqid=0 2024-11-07T09:36:16,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741927_1103 (size=14741) 2024-11-07T09:36:17,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:17,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972236997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:17,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:17,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:17,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:17,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:17,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972237000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:17,005 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:17,005 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:17,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:17,005 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:17,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972237000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:17,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:17,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:17,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:17,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:17,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972237103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:17,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:17,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:17,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:17,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:17,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972237112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:17,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:17,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:17,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:17,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:17,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972237113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:17,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:17,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:17,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:17,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:17,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972237313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:17,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:17,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:17,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:17,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:17,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972237322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:17,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:17,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:17,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:17,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:17,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972237322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:17,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:17,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:17,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:17,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-07T09:36:17,363 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:36:17,364 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T09:36:17,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-07T09:36:17,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-07T09:36:17,366 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T09:36:17,367 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T09:36:17,367 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T09:36:17,395 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/2d90565175e24ee69163e3dad96b830c 2024-11-07T09:36:17,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/86a433b552cb4a14ad2e91b998366890 is 50, key is test_row_0/B:col10/1730972176965/Put/seqid=0 2024-11-07T09:36:17,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741928_1104 (size=12301) 2024-11-07T09:36:17,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-07T09:36:17,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-07T09:36:17,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:17,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. as already flushing 2024-11-07T09:36:17,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:17,520 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:17,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:17,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:17,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:17,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972237623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:17,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:17,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:17,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:17,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:17,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972237633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:17,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:17,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:17,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:17,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:17,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972237633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:17,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:17,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:17,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:17,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-07T09:36:17,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:17,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. as already flushing 2024-11-07T09:36:17,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:17,674 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:17,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:17,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:17,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-07T09:36:17,810 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/86a433b552cb4a14ad2e91b998366890 2024-11-07T09:36:17,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/4874583fdc6c4b16960ab6422ae1fdfe is 50, key is test_row_0/C:col10/1730972176965/Put/seqid=0 2024-11-07T09:36:17,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741929_1105 (size=12301) 2024-11-07T09:36:17,825 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/4874583fdc6c4b16960ab6422ae1fdfe 2024-11-07T09:36:17,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-07T09:36:17,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:17,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. as already flushing 2024-11-07T09:36:17,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:17,828 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:17,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:17,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:17,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/2d90565175e24ee69163e3dad96b830c as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/2d90565175e24ee69163e3dad96b830c 2024-11-07T09:36:17,841 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/2d90565175e24ee69163e3dad96b830c, entries=200, sequenceid=447, filesize=14.4 K 2024-11-07T09:36:17,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/86a433b552cb4a14ad2e91b998366890 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/86a433b552cb4a14ad2e91b998366890 2024-11-07T09:36:17,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/86a433b552cb4a14ad2e91b998366890, entries=150, sequenceid=447, filesize=12.0 K 2024-11-07T09:36:17,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/4874583fdc6c4b16960ab6422ae1fdfe as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/4874583fdc6c4b16960ab6422ae1fdfe 2024-11-07T09:36:17,855 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/4874583fdc6c4b16960ab6422ae1fdfe, entries=150, sequenceid=447, filesize=12.0 K 2024-11-07T09:36:17,856 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 116dccc5098610cd38ab507cb3daad26 in 890ms, sequenceid=447, compaction requested=true 2024-11-07T09:36:17,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:17,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 116dccc5098610cd38ab507cb3daad26:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T09:36:17,856 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:36:17,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:17,857 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:36:17,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 116dccc5098610cd38ab507cb3daad26:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T09:36:17,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:17,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 116dccc5098610cd38ab507cb3daad26:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:36:17,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:36:17,859 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:36:17,859 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 116dccc5098610cd38ab507cb3daad26/B is initiating minor compaction (all files) 2024-11-07T09:36:17,859 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 116dccc5098610cd38ab507cb3daad26/B in TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:17,859 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/ff24f01394d34b8d90b1c4126c793358, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/068fd5c449dc4f70883e0bcbab28e6a7, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/86a433b552cb4a14ad2e91b998366890] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp, totalSize=37.0 K 2024-11-07T09:36:17,860 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40297 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:36:17,860 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting ff24f01394d34b8d90b1c4126c793358, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1730972173611 2024-11-07T09:36:17,860 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 116dccc5098610cd38ab507cb3daad26/A is initiating minor compaction (all files) 2024-11-07T09:36:17,860 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 116dccc5098610cd38ab507cb3daad26/A in TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:17,860 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/5081740c127642c8ac59ea97d09508a0, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/530961d9eb6e44bea45ef59833ce0166, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/2d90565175e24ee69163e3dad96b830c] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp, totalSize=39.4 K 2024-11-07T09:36:17,860 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 068fd5c449dc4f70883e0bcbab28e6a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1730972174795 2024-11-07T09:36:17,861 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5081740c127642c8ac59ea97d09508a0, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1730972173611 2024-11-07T09:36:17,861 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 86a433b552cb4a14ad2e91b998366890, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1730972176959 2024-11-07T09:36:17,861 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 530961d9eb6e44bea45ef59833ce0166, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1730972174795 2024-11-07T09:36:17,862 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d90565175e24ee69163e3dad96b830c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1730972176959 2024-11-07T09:36:17,873 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 116dccc5098610cd38ab507cb3daad26#B#compaction#93 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:17,875 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 116dccc5098610cd38ab507cb3daad26#A#compaction#94 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:17,875 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/3b696f4ae2ef4ec69d766bb51ec12119 is 50, key is test_row_0/B:col10/1730972176965/Put/seqid=0 2024-11-07T09:36:17,875 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/b4524744f63a47bcae0971726a59b256 is 50, key is test_row_0/A:col10/1730972176965/Put/seqid=0 2024-11-07T09:36:17,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741931_1107 (size=13357) 2024-11-07T09:36:17,898 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/b4524744f63a47bcae0971726a59b256 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/b4524744f63a47bcae0971726a59b256 2024-11-07T09:36:17,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741930_1106 (size=13357) 2024-11-07T09:36:17,908 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 116dccc5098610cd38ab507cb3daad26/A of 116dccc5098610cd38ab507cb3daad26 into b4524744f63a47bcae0971726a59b256(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:17,908 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:17,908 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., storeName=116dccc5098610cd38ab507cb3daad26/A, priority=13, startTime=1730972177856; duration=0sec 2024-11-07T09:36:17,908 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/3b696f4ae2ef4ec69d766bb51ec12119 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/3b696f4ae2ef4ec69d766bb51ec12119 2024-11-07T09:36:17,908 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:36:17,908 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 116dccc5098610cd38ab507cb3daad26:A 2024-11-07T09:36:17,908 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:36:17,910 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:36:17,910 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 116dccc5098610cd38ab507cb3daad26/C is initiating minor compaction (all files) 2024-11-07T09:36:17,910 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 116dccc5098610cd38ab507cb3daad26/C in TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:17,910 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/27b72e1e6ccc4ae2b3700ebdde578ea5, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/8d2099c6245741649815a0789c72d4dc, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/4874583fdc6c4b16960ab6422ae1fdfe] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp, totalSize=37.0 K 2024-11-07T09:36:17,911 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27b72e1e6ccc4ae2b3700ebdde578ea5, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1730972173611 2024-11-07T09:36:17,913 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d2099c6245741649815a0789c72d4dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1730972174795 2024-11-07T09:36:17,913 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4874583fdc6c4b16960ab6422ae1fdfe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1730972176959 2024-11-07T09:36:17,916 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 116dccc5098610cd38ab507cb3daad26/B of 116dccc5098610cd38ab507cb3daad26 into 3b696f4ae2ef4ec69d766bb51ec12119(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:17,916 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:17,916 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., storeName=116dccc5098610cd38ab507cb3daad26/B, priority=13, startTime=1730972177856; duration=0sec 2024-11-07T09:36:17,916 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:17,916 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 116dccc5098610cd38ab507cb3daad26:B 2024-11-07T09:36:17,924 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 116dccc5098610cd38ab507cb3daad26#C#compaction#95 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:17,925 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/8e5afecf7a5b4ce5814b702430157e39 is 50, key is test_row_0/C:col10/1730972176965/Put/seqid=0 2024-11-07T09:36:17,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741932_1108 (size=13357) 2024-11-07T09:36:17,946 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/8e5afecf7a5b4ce5814b702430157e39 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/8e5afecf7a5b4ce5814b702430157e39 2024-11-07T09:36:17,952 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 116dccc5098610cd38ab507cb3daad26/C of 116dccc5098610cd38ab507cb3daad26 into 8e5afecf7a5b4ce5814b702430157e39(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:17,953 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:17,953 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., storeName=116dccc5098610cd38ab507cb3daad26/C, priority=13, startTime=1730972177857; duration=0sec 2024-11-07T09:36:17,953 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:17,953 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 116dccc5098610cd38ab507cb3daad26:C 2024-11-07T09:36:17,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-07T09:36:17,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:17,983 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 116dccc5098610cd38ab507cb3daad26 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T09:36:17,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/8deb17a08c9b4dbf964ddb8d1c91207f is 50, key is test_row_0/A:col10/1730972176997/Put/seqid=0 2024-11-07T09:36:17,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-07T09:36:18,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741933_1109 (size=12301) 2024-11-07T09:36:18,003 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/8deb17a08c9b4dbf964ddb8d1c91207f 2024-11-07T09:36:18,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/df9bff6ddd5d4d3fabf3f98de3379b33 is 50, key is test_row_0/B:col10/1730972176997/Put/seqid=0 2024-11-07T09:36:18,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741934_1110 (size=12301) 2024-11-07T09:36:18,027 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/df9bff6ddd5d4d3fabf3f98de3379b33 2024-11-07T09:36:18,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/0b9135cefaa845b2bbe79f6a388f71b5 is 50, key is test_row_0/C:col10/1730972176997/Put/seqid=0 2024-11-07T09:36:18,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741935_1111 (size=12301) 2024-11-07T09:36:18,050 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/0b9135cefaa845b2bbe79f6a388f71b5 2024-11-07T09:36:18,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/8deb17a08c9b4dbf964ddb8d1c91207f as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/8deb17a08c9b4dbf964ddb8d1c91207f 2024-11-07T09:36:18,065 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/8deb17a08c9b4dbf964ddb8d1c91207f, entries=150, sequenceid=474, filesize=12.0 K 2024-11-07T09:36:18,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/df9bff6ddd5d4d3fabf3f98de3379b33 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/df9bff6ddd5d4d3fabf3f98de3379b33 2024-11-07T09:36:18,073 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/df9bff6ddd5d4d3fabf3f98de3379b33, entries=150, sequenceid=474, filesize=12.0 K 2024-11-07T09:36:18,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/0b9135cefaa845b2bbe79f6a388f71b5 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/0b9135cefaa845b2bbe79f6a388f71b5 2024-11-07T09:36:18,080 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/0b9135cefaa845b2bbe79f6a388f71b5, entries=150, sequenceid=474, filesize=12.0 K 2024-11-07T09:36:18,081 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=0 B/0 for 116dccc5098610cd38ab507cb3daad26 in 98ms, sequenceid=474, compaction requested=false 2024-11-07T09:36:18,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:18,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:18,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-07T09:36:18,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-07T09:36:18,086 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-07T09:36:18,086 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 716 msec 2024-11-07T09:36:18,088 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 723 msec 2024-11-07T09:36:18,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 116dccc5098610cd38ab507cb3daad26 2024-11-07T09:36:18,167 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:36:18,167 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 116dccc5098610cd38ab507cb3daad26 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T09:36:18,172 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/57a9322168194069be335e259b9f9e26 is 50, key is test_row_0/A:col10/1730972178154/Put/seqid=0 2024-11-07T09:36:18,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741936_1112 (size=12301) 2024-11-07T09:36:18,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:18,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972238207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:18,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:18,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:18,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:18,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:18,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972238210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:18,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:18,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:18,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:18,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:18,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972238210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:18,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:18,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:18,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:18,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:18,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972238322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:18,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:18,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:18,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972238322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:18,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:18,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:18,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:18,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:18,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:18,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:18,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972238322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:18,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:18,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:18,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:18,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-07T09:36:18,503 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:36:18,504 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T09:36:18,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-11-07T09:36:18,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-07T09:36:18,506 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T09:36:18,507 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T09:36:18,507 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T09:36:18,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:18,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972238533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:18,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:18,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:18,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:18,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:18,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972238533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:18,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:18,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:18,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972238533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:18,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:18,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:18,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:18,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:18,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:18,586 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/57a9322168194069be335e259b9f9e26 2024-11-07T09:36:18,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/289cb93affa54c7f8bc9255485ca8da0 is 50, key is test_row_0/B:col10/1730972178154/Put/seqid=0 2024-11-07T09:36:18,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741937_1113 (size=12301) 2024-11-07T09:36:18,600 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/289cb93affa54c7f8bc9255485ca8da0 2024-11-07T09:36:18,608 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/96240a0f86ac4d9aa072f0825b748543 is 50, key is test_row_0/C:col10/1730972178154/Put/seqid=0 2024-11-07T09:36:18,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-07T09:36:18,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741938_1114 (size=12301) 2024-11-07T09:36:18,616 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/96240a0f86ac4d9aa072f0825b748543 2024-11-07T09:36:18,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/57a9322168194069be335e259b9f9e26 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/57a9322168194069be335e259b9f9e26 2024-11-07T09:36:18,627 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/57a9322168194069be335e259b9f9e26, entries=150, sequenceid=485, filesize=12.0 K 2024-11-07T09:36:18,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/289cb93affa54c7f8bc9255485ca8da0 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/289cb93affa54c7f8bc9255485ca8da0 2024-11-07T09:36:18,633 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/289cb93affa54c7f8bc9255485ca8da0, entries=150, sequenceid=485, filesize=12.0 K 2024-11-07T09:36:18,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/96240a0f86ac4d9aa072f0825b748543 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/96240a0f86ac4d9aa072f0825b748543 2024-11-07T09:36:18,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/96240a0f86ac4d9aa072f0825b748543, entries=150, sequenceid=485, filesize=12.0 K 2024-11-07T09:36:18,641 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 116dccc5098610cd38ab507cb3daad26 in 474ms, sequenceid=485, compaction requested=true 2024-11-07T09:36:18,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:18,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 116dccc5098610cd38ab507cb3daad26:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T09:36:18,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:18,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 116dccc5098610cd38ab507cb3daad26:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T09:36:18,641 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:36:18,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:18,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 116dccc5098610cd38ab507cb3daad26:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:36:18,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:36:18,641 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:36:18,642 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:36:18,642 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 116dccc5098610cd38ab507cb3daad26/A is initiating minor compaction (all files) 2024-11-07T09:36:18,642 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:36:18,643 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 116dccc5098610cd38ab507cb3daad26/A in TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:18,643 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 116dccc5098610cd38ab507cb3daad26/B is initiating minor compaction (all files) 2024-11-07T09:36:18,643 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 116dccc5098610cd38ab507cb3daad26/B in TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:18,643 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/b4524744f63a47bcae0971726a59b256, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/8deb17a08c9b4dbf964ddb8d1c91207f, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/57a9322168194069be335e259b9f9e26] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp, totalSize=37.1 K 2024-11-07T09:36:18,643 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/3b696f4ae2ef4ec69d766bb51ec12119, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/df9bff6ddd5d4d3fabf3f98de3379b33, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/289cb93affa54c7f8bc9255485ca8da0] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp, totalSize=37.1 K 2024-11-07T09:36:18,643 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4524744f63a47bcae0971726a59b256, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1730972176959 2024-11-07T09:36:18,643 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b696f4ae2ef4ec69d766bb51ec12119, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1730972176959 2024-11-07T09:36:18,644 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8deb17a08c9b4dbf964ddb8d1c91207f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1730972176985 2024-11-07T09:36:18,644 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting df9bff6ddd5d4d3fabf3f98de3379b33, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1730972176985 2024-11-07T09:36:18,644 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57a9322168194069be335e259b9f9e26, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1730972178154 2024-11-07T09:36:18,644 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 289cb93affa54c7f8bc9255485ca8da0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1730972178154 2024-11-07T09:36:18,653 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 116dccc5098610cd38ab507cb3daad26#A#compaction#102 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:18,653 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 116dccc5098610cd38ab507cb3daad26#B#compaction#103 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:18,653 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/7fe787f0edbe4d8e887bcf8c9c13f61c is 50, key is test_row_0/A:col10/1730972178154/Put/seqid=0 2024-11-07T09:36:18,654 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/d578fe4e0ccd4736b7abe4bdd59eb148 is 50, key is test_row_0/B:col10/1730972178154/Put/seqid=0 2024-11-07T09:36:18,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-07T09:36:18,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:18,659 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing 116dccc5098610cd38ab507cb3daad26 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T09:36:18,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741940_1116 (size=13459) 2024-11-07T09:36:18,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/0f57baf32f7e4011837b287364dbc70d is 50, key is test_row_0/A:col10/1730972178205/Put/seqid=0 2024-11-07T09:36:18,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741939_1115 (size=13459) 2024-11-07T09:36:18,673 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/d578fe4e0ccd4736b7abe4bdd59eb148 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/d578fe4e0ccd4736b7abe4bdd59eb148 2024-11-07T09:36:18,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741941_1117 (size=12301) 2024-11-07T09:36:18,681 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 116dccc5098610cd38ab507cb3daad26/B of 116dccc5098610cd38ab507cb3daad26 into d578fe4e0ccd4736b7abe4bdd59eb148(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:18,681 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:18,681 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., storeName=116dccc5098610cd38ab507cb3daad26/B, priority=13, startTime=1730972178641; duration=0sec 2024-11-07T09:36:18,682 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:36:18,682 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 116dccc5098610cd38ab507cb3daad26:B 2024-11-07T09:36:18,682 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:36:18,683 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:36:18,683 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 116dccc5098610cd38ab507cb3daad26/C is initiating minor compaction (all files) 2024-11-07T09:36:18,684 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 116dccc5098610cd38ab507cb3daad26/C in TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:18,684 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/8e5afecf7a5b4ce5814b702430157e39, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/0b9135cefaa845b2bbe79f6a388f71b5, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/96240a0f86ac4d9aa072f0825b748543] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp, totalSize=37.1 K 2024-11-07T09:36:18,684 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e5afecf7a5b4ce5814b702430157e39, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1730972176959 2024-11-07T09:36:18,685 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b9135cefaa845b2bbe79f6a388f71b5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1730972176985 2024-11-07T09:36:18,685 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 96240a0f86ac4d9aa072f0825b748543, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1730972178154 2024-11-07T09:36:18,693 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 116dccc5098610cd38ab507cb3daad26#C#compaction#105 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:18,693 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/7001642959754d16b89ab32d65ad1590 is 50, key is test_row_0/C:col10/1730972178154/Put/seqid=0 2024-11-07T09:36:18,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741942_1118 (size=13459) 2024-11-07T09:36:18,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-07T09:36:18,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 116dccc5098610cd38ab507cb3daad26 2024-11-07T09:36:18,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. as already flushing 2024-11-07T09:36:18,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:18,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:18,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972238853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:18,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972238852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:18,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:18,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:18,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:18,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:18,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:18,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:18,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:18,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972238857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:18,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:18,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:18,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:18,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:18,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:18,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972238962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:18,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972238962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:18,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:18,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:18,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:18,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:18,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:18,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:18,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:18,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972238962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:18,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:18,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:18,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:19,074 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=510 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/0f57baf32f7e4011837b287364dbc70d 2024-11-07T09:36:19,076 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/7fe787f0edbe4d8e887bcf8c9c13f61c as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/7fe787f0edbe4d8e887bcf8c9c13f61c 2024-11-07T09:36:19,082 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 116dccc5098610cd38ab507cb3daad26/A of 116dccc5098610cd38ab507cb3daad26 into 7fe787f0edbe4d8e887bcf8c9c13f61c(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:19,082 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:19,082 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., storeName=116dccc5098610cd38ab507cb3daad26/A, priority=13, startTime=1730972178641; duration=0sec 2024-11-07T09:36:19,083 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:19,083 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 116dccc5098610cd38ab507cb3daad26:A 2024-11-07T09:36:19,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/01704f981b7b42a087dfcab4c156be1b is 50, key is test_row_0/B:col10/1730972178205/Put/seqid=0 2024-11-07T09:36:19,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741943_1119 (size=12301) 2024-11-07T09:36:19,108 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/7001642959754d16b89ab32d65ad1590 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/7001642959754d16b89ab32d65ad1590 2024-11-07T09:36:19,113 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 116dccc5098610cd38ab507cb3daad26/C of 116dccc5098610cd38ab507cb3daad26 into 7001642959754d16b89ab32d65ad1590(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:19,113 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:19,113 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., storeName=116dccc5098610cd38ab507cb3daad26/C, priority=13, startTime=1730972178641; duration=0sec 2024-11-07T09:36:19,113 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:19,113 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 116dccc5098610cd38ab507cb3daad26:C 2024-11-07T09:36:19,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-07T09:36:19,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:19,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972239173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:19,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:19,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:19,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:19,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:19,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:19,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972239173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:19,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972239174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:19,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:19,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:19,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:19,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:19,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:19,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:19,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:19,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972239483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:19,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:19,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:19,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:19,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:19,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972239483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:19,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:19,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:19,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:19,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:19,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972239485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:19,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:19,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:19,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:19,490 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=510 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/01704f981b7b42a087dfcab4c156be1b 2024-11-07T09:36:19,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/3b7c2318507f4f00bcdd17ee40d622ba is 50, key is test_row_0/C:col10/1730972178205/Put/seqid=0 2024-11-07T09:36:19,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741944_1120 (size=12301) 2024-11-07T09:36:19,511 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=510 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/3b7c2318507f4f00bcdd17ee40d622ba 2024-11-07T09:36:19,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/0f57baf32f7e4011837b287364dbc70d as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/0f57baf32f7e4011837b287364dbc70d 2024-11-07T09:36:19,526 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/0f57baf32f7e4011837b287364dbc70d, entries=150, sequenceid=510, filesize=12.0 K 2024-11-07T09:36:19,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/01704f981b7b42a087dfcab4c156be1b as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/01704f981b7b42a087dfcab4c156be1b 2024-11-07T09:36:19,533 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/01704f981b7b42a087dfcab4c156be1b, entries=150, sequenceid=510, filesize=12.0 K 2024-11-07T09:36:19,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/3b7c2318507f4f00bcdd17ee40d622ba as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/3b7c2318507f4f00bcdd17ee40d622ba 2024-11-07T09:36:19,539 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/3b7c2318507f4f00bcdd17ee40d622ba, entries=150, sequenceid=510, filesize=12.0 K 2024-11-07T09:36:19,540 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 116dccc5098610cd38ab507cb3daad26 in 881ms, sequenceid=510, compaction requested=false 2024-11-07T09:36:19,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:19,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:19,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-11-07T09:36:19,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-11-07T09:36:19,546 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-07T09:36:19,546 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0360 sec 2024-11-07T09:36:19,548 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 1.0420 sec 2024-11-07T09:36:19,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-07T09:36:19,643 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:36:19,643 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T09:36:19,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees 2024-11-07T09:36:19,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-07T09:36:19,646 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=29, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T09:36:19,646 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=29, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T09:36:19,647 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T09:36:19,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-07T09:36:19,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-07T09:36:19,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:19,800 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2902): Flushing 116dccc5098610cd38ab507cb3daad26 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T09:36:19,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/d37be101ee83498fbd1ec23b340b06c4 is 50, key is test_row_0/A:col10/1730972178852/Put/seqid=0 2024-11-07T09:36:19,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741945_1121 (size=12301) 2024-11-07T09:36:19,811 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=524 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/d37be101ee83498fbd1ec23b340b06c4 2024-11-07T09:36:19,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/6d4e3442476e40d2af5935a77535cd26 is 50, key is test_row_0/B:col10/1730972178852/Put/seqid=0 2024-11-07T09:36:19,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741946_1122 (size=12301) 2024-11-07T09:36:19,841 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=524 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/6d4e3442476e40d2af5935a77535cd26 2024-11-07T09:36:19,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/2f29f1ea4b6e4562bddf81bd92a26554 is 50, key is test_row_0/C:col10/1730972178852/Put/seqid=0 2024-11-07T09:36:19,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741947_1123 (size=12301) 2024-11-07T09:36:19,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-07T09:36:19,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 116dccc5098610cd38ab507cb3daad26 2024-11-07T09:36:19,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. as already flushing 2024-11-07T09:36:20,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:20,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:20,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972240027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:20,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972240028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:20,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:20,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:20,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:20,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:20,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972240030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:20,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:20,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:20,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:20,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:20,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:20,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:20,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:20,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972240142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:20,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:20,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:20,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:20,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972240142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:20,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:20,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:20,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:20,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:20,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972240143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:20,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:20,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:20,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:20,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:20,257 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=524 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/2f29f1ea4b6e4562bddf81bd92a26554 2024-11-07T09:36:20,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/d37be101ee83498fbd1ec23b340b06c4 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/d37be101ee83498fbd1ec23b340b06c4 2024-11-07T09:36:20,268 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/d37be101ee83498fbd1ec23b340b06c4, entries=150, sequenceid=524, filesize=12.0 K 2024-11-07T09:36:20,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/6d4e3442476e40d2af5935a77535cd26 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/6d4e3442476e40d2af5935a77535cd26 2024-11-07T09:36:20,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-07T09:36:20,277 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/6d4e3442476e40d2af5935a77535cd26, entries=150, sequenceid=524, filesize=12.0 K 2024-11-07T09:36:20,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/2f29f1ea4b6e4562bddf81bd92a26554 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/2f29f1ea4b6e4562bddf81bd92a26554 2024-11-07T09:36:20,283 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/2f29f1ea4b6e4562bddf81bd92a26554, entries=150, sequenceid=524, filesize=12.0 K 2024-11-07T09:36:20,284 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 116dccc5098610cd38ab507cb3daad26 in 484ms, sequenceid=524, compaction requested=true 2024-11-07T09:36:20,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2603): Flush status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:20,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:20,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=30 2024-11-07T09:36:20,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4169): Remote procedure done, pid=30 2024-11-07T09:36:20,289 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-07T09:36:20,289 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 640 msec 2024-11-07T09:36:20,291 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees in 647 msec 2024-11-07T09:36:20,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 116dccc5098610cd38ab507cb3daad26 2024-11-07T09:36:20,357 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:36:20,357 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 116dccc5098610cd38ab507cb3daad26 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-07T09:36:20,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/7de168851516462ea018c70ce97092c1 is 50, key is test_row_0/A:col10/1730972180355/Put/seqid=0 2024-11-07T09:36:20,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741948_1124 (size=12301) 2024-11-07T09:36:20,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:20,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972240369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:20,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:20,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972240371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:20,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:20,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:20,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:20,373 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:20,373 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:20,373 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:20,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:20,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972240371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:20,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:20,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:20,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:20,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:20,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:20,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972240483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:20,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972240483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:20,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:20,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972240483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:20,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:20,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:20,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:20,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:20,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:20,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:20,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:20,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:20,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:20,512 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-07T09:36:20,512 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-07T09:36:20,512 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:36:20,512 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:36:20,514 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-07T09:36:20,514 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:36:20,515 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-07T09:36:20,515 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:36:20,518 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:20,518 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:20,518 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:20,518 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:20,518 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:20,518 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:20,518 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:20,518 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:20,519 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:36:20,519 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:36:20,518 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:36:20,519 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:36:20,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:20,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:20,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:20,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972240693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:20,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972240693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:20,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972240693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:20,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:20,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:20,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:20,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:20,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:20,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:20,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:20,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:20,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:20,771 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=552 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/7de168851516462ea018c70ce97092c1 2024-11-07T09:36:20,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/d4c6400943d84010a07669eae3cae720 is 50, key is test_row_0/B:col10/1730972180355/Put/seqid=0 2024-11-07T09:36:20,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-07T09:36:20,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741949_1125 (size=12301) 2024-11-07T09:36:20,783 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:36:21,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:21,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:21,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:21,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972241003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:21,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972241003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:21,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972241003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:21,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:21,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:21,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:21,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:21,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:21,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:21,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:21,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:21,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:21,184 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=552 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/d4c6400943d84010a07669eae3cae720 2024-11-07T09:36:21,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/8d3ec6a1e574497c8aba11cf592c679c is 50, key is test_row_0/C:col10/1730972180355/Put/seqid=0 2024-11-07T09:36:21,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741950_1126 (size=12301) 2024-11-07T09:36:21,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:21,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:21,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:21,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33450 deadline: 1730972241513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:21,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1730972241513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:21,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33492 deadline: 1730972241513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:21,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:21,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:21,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:21,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:21,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:21,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:21,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:21,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:21,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:36:21,596 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=552 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/8d3ec6a1e574497c8aba11cf592c679c 2024-11-07T09:36:21,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/7de168851516462ea018c70ce97092c1 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/7de168851516462ea018c70ce97092c1 2024-11-07T09:36:21,606 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/7de168851516462ea018c70ce97092c1, entries=150, sequenceid=552, filesize=12.0 K 2024-11-07T09:36:21,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/d4c6400943d84010a07669eae3cae720 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/d4c6400943d84010a07669eae3cae720 2024-11-07T09:36:21,611 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/d4c6400943d84010a07669eae3cae720, entries=150, sequenceid=552, filesize=12.0 K 2024-11-07T09:36:21,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/8d3ec6a1e574497c8aba11cf592c679c as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/8d3ec6a1e574497c8aba11cf592c679c 2024-11-07T09:36:21,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/8d3ec6a1e574497c8aba11cf592c679c, entries=150, sequenceid=552, filesize=12.0 K 2024-11-07T09:36:21,618 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=40.25 KB/41220 for 116dccc5098610cd38ab507cb3daad26 in 1260ms, sequenceid=552, compaction requested=true 2024-11-07T09:36:21,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:21,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 116dccc5098610cd38ab507cb3daad26:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T09:36:21,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:21,618 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T09:36:21,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 116dccc5098610cd38ab507cb3daad26:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T09:36:21,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:21,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 116dccc5098610cd38ab507cb3daad26:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:36:21,618 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T09:36:21,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:36:21,619 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50362 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T09:36:21,619 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50362 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T09:36:21,619 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 116dccc5098610cd38ab507cb3daad26/A is initiating minor compaction (all files) 2024-11-07T09:36:21,619 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 116dccc5098610cd38ab507cb3daad26/B is initiating minor compaction (all files) 2024-11-07T09:36:21,619 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 116dccc5098610cd38ab507cb3daad26/A in TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:21,619 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 116dccc5098610cd38ab507cb3daad26/B in TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:21,619 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/7fe787f0edbe4d8e887bcf8c9c13f61c, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/0f57baf32f7e4011837b287364dbc70d, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/d37be101ee83498fbd1ec23b340b06c4, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/7de168851516462ea018c70ce97092c1] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp, totalSize=49.2 K 2024-11-07T09:36:21,619 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/d578fe4e0ccd4736b7abe4bdd59eb148, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/01704f981b7b42a087dfcab4c156be1b, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/6d4e3442476e40d2af5935a77535cd26, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/d4c6400943d84010a07669eae3cae720] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp, totalSize=49.2 K 2024-11-07T09:36:21,620 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting d578fe4e0ccd4736b7abe4bdd59eb148, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1730972178154 2024-11-07T09:36:21,620 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fe787f0edbe4d8e887bcf8c9c13f61c, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1730972178154 2024-11-07T09:36:21,620 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 01704f981b7b42a087dfcab4c156be1b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=510, earliestPutTs=1730972178205 2024-11-07T09:36:21,620 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f57baf32f7e4011837b287364dbc70d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=510, earliestPutTs=1730972178205 2024-11-07T09:36:21,621 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d4e3442476e40d2af5935a77535cd26, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=524, earliestPutTs=1730972178848 2024-11-07T09:36:21,621 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting d37be101ee83498fbd1ec23b340b06c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=524, earliestPutTs=1730972178848 2024-11-07T09:36:21,621 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting d4c6400943d84010a07669eae3cae720, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=552, earliestPutTs=1730972180026 2024-11-07T09:36:21,621 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7de168851516462ea018c70ce97092c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=552, earliestPutTs=1730972180026 2024-11-07T09:36:21,630 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 116dccc5098610cd38ab507cb3daad26#A#compaction#114 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:21,631 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/eb09b9e33b2c447db390473f1c1aa3fa is 50, key is test_row_0/A:col10/1730972180355/Put/seqid=0 2024-11-07T09:36:21,631 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 116dccc5098610cd38ab507cb3daad26#B#compaction#115 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:21,632 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/f43ab72b640246fda5c15d588c2eebb2 is 50, key is test_row_0/B:col10/1730972180355/Put/seqid=0 2024-11-07T09:36:21,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741951_1127 (size=13595) 2024-11-07T09:36:21,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741952_1128 (size=13595) 2024-11-07T09:36:22,042 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/eb09b9e33b2c447db390473f1c1aa3fa as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/eb09b9e33b2c447db390473f1c1aa3fa 2024-11-07T09:36:22,043 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/f43ab72b640246fda5c15d588c2eebb2 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/f43ab72b640246fda5c15d588c2eebb2 2024-11-07T09:36:22,047 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 116dccc5098610cd38ab507cb3daad26/A of 116dccc5098610cd38ab507cb3daad26 into eb09b9e33b2c447db390473f1c1aa3fa(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:22,047 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:22,047 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., storeName=116dccc5098610cd38ab507cb3daad26/A, priority=12, startTime=1730972181618; duration=0sec 2024-11-07T09:36:22,048 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:36:22,048 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 116dccc5098610cd38ab507cb3daad26:A 2024-11-07T09:36:22,048 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T09:36:22,048 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 116dccc5098610cd38ab507cb3daad26/B of 116dccc5098610cd38ab507cb3daad26 into f43ab72b640246fda5c15d588c2eebb2(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:22,048 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:22,048 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., storeName=116dccc5098610cd38ab507cb3daad26/B, priority=12, startTime=1730972181618; duration=0sec 2024-11-07T09:36:22,048 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:22,048 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 116dccc5098610cd38ab507cb3daad26:B 2024-11-07T09:36:22,049 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50362 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T09:36:22,049 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 116dccc5098610cd38ab507cb3daad26/C is initiating minor compaction (all files) 2024-11-07T09:36:22,049 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 116dccc5098610cd38ab507cb3daad26/C in TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:22,049 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/7001642959754d16b89ab32d65ad1590, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/3b7c2318507f4f00bcdd17ee40d622ba, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/2f29f1ea4b6e4562bddf81bd92a26554, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/8d3ec6a1e574497c8aba11cf592c679c] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp, totalSize=49.2 K 2024-11-07T09:36:22,050 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7001642959754d16b89ab32d65ad1590, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1730972178154 2024-11-07T09:36:22,050 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b7c2318507f4f00bcdd17ee40d622ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=510, earliestPutTs=1730972178205 2024-11-07T09:36:22,050 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f29f1ea4b6e4562bddf81bd92a26554, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=524, earliestPutTs=1730972178848 2024-11-07T09:36:22,051 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d3ec6a1e574497c8aba11cf592c679c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=552, earliestPutTs=1730972180026 2024-11-07T09:36:22,059 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 116dccc5098610cd38ab507cb3daad26#C#compaction#116 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:22,059 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/3028693de8a84d76ad79e08d2abbb4a1 is 50, key is test_row_0/C:col10/1730972180355/Put/seqid=0 2024-11-07T09:36:22,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741953_1129 (size=13595) 2024-11-07T09:36:22,442 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T09:36:22,469 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/3028693de8a84d76ad79e08d2abbb4a1 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/3028693de8a84d76ad79e08d2abbb4a1 2024-11-07T09:36:22,475 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 116dccc5098610cd38ab507cb3daad26/C of 116dccc5098610cd38ab507cb3daad26 into 3028693de8a84d76ad79e08d2abbb4a1(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:22,475 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:22,475 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., storeName=116dccc5098610cd38ab507cb3daad26/C, priority=12, startTime=1730972181618; duration=0sec 2024-11-07T09:36:22,475 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:22,475 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 116dccc5098610cd38ab507cb3daad26:C 2024-11-07T09:36:22,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 116dccc5098610cd38ab507cb3daad26 2024-11-07T09:36:22,526 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-07T09:36:22,526 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:36:22,526 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:22,527 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:22,527 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:36:22,527 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-07T09:36:22,527 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 116dccc5098610cd38ab507cb3daad26 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-07T09:36:22,527 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:36:22,527 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:36:22,527 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:22,527 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:22,527 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:36:22,527 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-07T09:36:22,527 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:36:22,527 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:22,527 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:22,527 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:36:22,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/df599fbc21e142de8c670c60920dec23 is 50, key is test_row_0/A:col10/1730972182525/Put/seqid=0 2024-11-07T09:36:22,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741954_1130 (size=12301) 2024-11-07T09:36:22,937 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=567 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/df599fbc21e142de8c670c60920dec23 2024-11-07T09:36:22,946 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/74a7adf09e5c4262aa893411cbd3e48a is 50, key is test_row_0/B:col10/1730972182525/Put/seqid=0 2024-11-07T09:36:22,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741955_1131 (size=12301) 2024-11-07T09:36:23,352 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=567 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/74a7adf09e5c4262aa893411cbd3e48a 2024-11-07T09:36:23,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/2170211cde6e43e9b2b4acbc0d2c90c5 is 50, key is test_row_0/C:col10/1730972182525/Put/seqid=0 2024-11-07T09:36:23,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741956_1132 (size=12301) 2024-11-07T09:36:23,675 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-07T09:36:23,675 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-07T09:36:23,675 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:36:23,675 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:36:23,675 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:23,675 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:23,675 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:23,675 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:23,675 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:36:23,675 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:36:23,675 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-07T09:36:23,675 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 127 2024-11-07T09:36:23,675 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 123 2024-11-07T09:36:23,675 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 123 2024-11-07T09:36:23,675 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-11-07T09:36:23,675 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 37 2024-11-07T09:36:23,675 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-07T09:36:23,676 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3356 2024-11-07T09:36:23,676 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3329 2024-11-07T09:36:23,676 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-07T09:36:23,676 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1532 2024-11-07T09:36:23,676 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4577 rows 2024-11-07T09:36:23,676 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1533 2024-11-07T09:36:23,676 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4587 rows 2024-11-07T09:36:23,676 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T09:36:23,676 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T09:36:23,676 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:23,676 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:23,676 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:36:23,681 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-07T09:36:23,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-07T09:36:23,687 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730972183687"}]},"ts":"1730972183687"} 2024-11-07T09:36:23,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-07T09:36:23,689 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-07T09:36:23,689 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-07T09:36:23,691 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T09:36:23,694 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=116dccc5098610cd38ab507cb3daad26, UNASSIGN}] 2024-11-07T09:36:23,695 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=116dccc5098610cd38ab507cb3daad26, UNASSIGN 2024-11-07T09:36:23,696 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=116dccc5098610cd38ab507cb3daad26, regionState=CLOSING, regionLocation=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:23,698 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=116dccc5098610cd38ab507cb3daad26, UNASSIGN because future has completed 2024-11-07T09:36:23,699 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T09:36:23,699 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970}] 2024-11-07T09:36:23,765 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=567 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/2170211cde6e43e9b2b4acbc0d2c90c5 2024-11-07T09:36:23,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/df599fbc21e142de8c670c60920dec23 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/df599fbc21e142de8c670c60920dec23 2024-11-07T09:36:23,776 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/df599fbc21e142de8c670c60920dec23, entries=150, sequenceid=567, filesize=12.0 K 2024-11-07T09:36:23,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/74a7adf09e5c4262aa893411cbd3e48a as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/74a7adf09e5c4262aa893411cbd3e48a 2024-11-07T09:36:23,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/74a7adf09e5c4262aa893411cbd3e48a, entries=150, sequenceid=567, filesize=12.0 K 2024-11-07T09:36:23,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/2170211cde6e43e9b2b4acbc0d2c90c5 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/2170211cde6e43e9b2b4acbc0d2c90c5 2024-11-07T09:36:23,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/2170211cde6e43e9b2b4acbc0d2c90c5, entries=150, sequenceid=567, filesize=12.0 K 2024-11-07T09:36:23,788 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=13.42 KB/13740 for 116dccc5098610cd38ab507cb3daad26 in 1261ms, sequenceid=567, compaction requested=false 2024-11-07T09:36:23,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 116dccc5098610cd38ab507cb3daad26: 2024-11-07T09:36:23,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-07T09:36:23,856 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(122): Close 116dccc5098610cd38ab507cb3daad26 2024-11-07T09:36:23,856 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T09:36:23,856 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1722): Closing 116dccc5098610cd38ab507cb3daad26, disabling compactions & flushes 2024-11-07T09:36:23,856 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:23,856 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:23,857 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. after waiting 0 ms 2024-11-07T09:36:23,857 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:23,857 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(2902): Flushing 116dccc5098610cd38ab507cb3daad26 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-07T09:36:23,861 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/1fd111c99880444287bad7cc0214afae is 50, key is test_row_1/A:col10/1730972183673/Put/seqid=0 2024-11-07T09:36:23,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741957_1133 (size=9857) 2024-11-07T09:36:24,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-07T09:36:24,266 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=572 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/1fd111c99880444287bad7cc0214afae 2024-11-07T09:36:24,274 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/1c8f48077ced43159e2643d987dfbaf0 is 50, key is test_row_1/B:col10/1730972183673/Put/seqid=0 2024-11-07T09:36:24,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741958_1134 (size=9857) 2024-11-07T09:36:24,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-07T09:36:24,678 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=572 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/1c8f48077ced43159e2643d987dfbaf0 2024-11-07T09:36:24,686 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/856bcbc0e30a40f1ae5c1dc86e81d4ee is 50, key is test_row_1/C:col10/1730972183673/Put/seqid=0 2024-11-07T09:36:24,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741959_1135 (size=9857) 2024-11-07T09:36:24,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-07T09:36:25,091 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=572 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/856bcbc0e30a40f1ae5c1dc86e81d4ee 2024-11-07T09:36:25,096 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/A/1fd111c99880444287bad7cc0214afae as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/1fd111c99880444287bad7cc0214afae 2024-11-07T09:36:25,101 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/1fd111c99880444287bad7cc0214afae, entries=100, sequenceid=572, filesize=9.6 K 2024-11-07T09:36:25,101 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/B/1c8f48077ced43159e2643d987dfbaf0 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/1c8f48077ced43159e2643d987dfbaf0 2024-11-07T09:36:25,106 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/1c8f48077ced43159e2643d987dfbaf0, entries=100, sequenceid=572, filesize=9.6 K 2024-11-07T09:36:25,106 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/.tmp/C/856bcbc0e30a40f1ae5c1dc86e81d4ee as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/856bcbc0e30a40f1ae5c1dc86e81d4ee 2024-11-07T09:36:25,111 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/856bcbc0e30a40f1ae5c1dc86e81d4ee, entries=100, sequenceid=572, filesize=9.6 K 2024-11-07T09:36:25,111 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 116dccc5098610cd38ab507cb3daad26 in 1254ms, sequenceid=572, compaction requested=true 2024-11-07T09:36:25,112 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/e395db610c4e453cb184045ae4293e78, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/1927300554a54fe6abb33ee9a6a45d71, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/8fbc2603ac0f4a5fb7ee6150720b56fd, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/86ec91df06724153b224b65a4d52b582, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/ce0be2dcec52464295a84c717ebeeee8, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/df505b5175df4652a5caa4fa80a960b8, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/5529f3fb6df5466283cb3183ee25ffa8, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/6e734765090e429bbd775e73cf5777b2, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/579dfee71e374768a9b117c8cd0aaf02, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/96eaa21aed454bfeb10d3581644e6a5d, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/699a85f190a240b787d2cdf01d3777a6, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/67e95a76c13d4b0cb4e869683a00b74c, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/9ae015902c314504b07dbe02b3621147, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/2819a6c970ee43b4856f9a9af12a2ad9, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/345f728ae70d4b7db7322a9868e4b5ed, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/0e1769cdf0174fcb9ae9ffbc518ee4a6, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/c26bf73380a5467b9d5ac64f99b71463, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/8524df10b8d141d6839c633dc70bd3ad, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/9973be5373334892a5704b8f0a63f8b4, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/2adec459751c44a690d4d0db75729f69, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/bd6abe3fc8ca4682a850015d73b117a9, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/b6e017b923dc478a92b2daed0c2a58c3, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/821882ae46ef4a4bb4e7f3e450ffe4cc, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/38c0949cdd214f83b07167e472905239, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/e2ed47e7eab343f497619c1f23cff3cf, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/1ff56655b1b544fb959350edef75f74c, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/0ee80eee76e644c288f3f4a350e36354, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/5081740c127642c8ac59ea97d09508a0, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/a35fd9ed4dfe45bf997f43d535c79f20, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/530961d9eb6e44bea45ef59833ce0166, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/2d90565175e24ee69163e3dad96b830c, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/b4524744f63a47bcae0971726a59b256, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/8deb17a08c9b4dbf964ddb8d1c91207f, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/7fe787f0edbe4d8e887bcf8c9c13f61c, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/57a9322168194069be335e259b9f9e26, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/0f57baf32f7e4011837b287364dbc70d, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/d37be101ee83498fbd1ec23b340b06c4, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/7de168851516462ea018c70ce97092c1] to archive 2024-11-07T09:36:25,115 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T09:36:25,120 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/e395db610c4e453cb184045ae4293e78 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/e395db610c4e453cb184045ae4293e78 2024-11-07T09:36:25,122 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/1927300554a54fe6abb33ee9a6a45d71 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/1927300554a54fe6abb33ee9a6a45d71 2024-11-07T09:36:25,123 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/8fbc2603ac0f4a5fb7ee6150720b56fd to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/8fbc2603ac0f4a5fb7ee6150720b56fd 2024-11-07T09:36:25,124 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/86ec91df06724153b224b65a4d52b582 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/86ec91df06724153b224b65a4d52b582 2024-11-07T09:36:25,126 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/ce0be2dcec52464295a84c717ebeeee8 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/ce0be2dcec52464295a84c717ebeeee8 2024-11-07T09:36:25,127 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/df505b5175df4652a5caa4fa80a960b8 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/df505b5175df4652a5caa4fa80a960b8 2024-11-07T09:36:25,128 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/5529f3fb6df5466283cb3183ee25ffa8 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/5529f3fb6df5466283cb3183ee25ffa8 2024-11-07T09:36:25,129 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/6e734765090e429bbd775e73cf5777b2 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/6e734765090e429bbd775e73cf5777b2 2024-11-07T09:36:25,130 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/579dfee71e374768a9b117c8cd0aaf02 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/579dfee71e374768a9b117c8cd0aaf02 2024-11-07T09:36:25,131 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/96eaa21aed454bfeb10d3581644e6a5d to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/96eaa21aed454bfeb10d3581644e6a5d 2024-11-07T09:36:25,132 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/699a85f190a240b787d2cdf01d3777a6 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/699a85f190a240b787d2cdf01d3777a6 2024-11-07T09:36:25,133 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/67e95a76c13d4b0cb4e869683a00b74c to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/67e95a76c13d4b0cb4e869683a00b74c 2024-11-07T09:36:25,134 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/9ae015902c314504b07dbe02b3621147 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/9ae015902c314504b07dbe02b3621147 2024-11-07T09:36:25,136 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/2819a6c970ee43b4856f9a9af12a2ad9 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/2819a6c970ee43b4856f9a9af12a2ad9 2024-11-07T09:36:25,137 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/345f728ae70d4b7db7322a9868e4b5ed to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/345f728ae70d4b7db7322a9868e4b5ed 2024-11-07T09:36:25,138 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/0e1769cdf0174fcb9ae9ffbc518ee4a6 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/0e1769cdf0174fcb9ae9ffbc518ee4a6 2024-11-07T09:36:25,139 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/c26bf73380a5467b9d5ac64f99b71463 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/c26bf73380a5467b9d5ac64f99b71463 2024-11-07T09:36:25,140 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/8524df10b8d141d6839c633dc70bd3ad to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/8524df10b8d141d6839c633dc70bd3ad 2024-11-07T09:36:25,141 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/9973be5373334892a5704b8f0a63f8b4 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/9973be5373334892a5704b8f0a63f8b4 2024-11-07T09:36:25,143 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/2adec459751c44a690d4d0db75729f69 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/2adec459751c44a690d4d0db75729f69 2024-11-07T09:36:25,144 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/bd6abe3fc8ca4682a850015d73b117a9 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/bd6abe3fc8ca4682a850015d73b117a9 2024-11-07T09:36:25,145 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/b6e017b923dc478a92b2daed0c2a58c3 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/b6e017b923dc478a92b2daed0c2a58c3 2024-11-07T09:36:25,146 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/821882ae46ef4a4bb4e7f3e450ffe4cc to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/821882ae46ef4a4bb4e7f3e450ffe4cc 2024-11-07T09:36:25,147 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/38c0949cdd214f83b07167e472905239 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/38c0949cdd214f83b07167e472905239 2024-11-07T09:36:25,148 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/e2ed47e7eab343f497619c1f23cff3cf to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/e2ed47e7eab343f497619c1f23cff3cf 2024-11-07T09:36:25,150 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/1ff56655b1b544fb959350edef75f74c to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/1ff56655b1b544fb959350edef75f74c 2024-11-07T09:36:25,151 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/0ee80eee76e644c288f3f4a350e36354 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/0ee80eee76e644c288f3f4a350e36354 2024-11-07T09:36:25,152 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/5081740c127642c8ac59ea97d09508a0 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/5081740c127642c8ac59ea97d09508a0 2024-11-07T09:36:25,154 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/a35fd9ed4dfe45bf997f43d535c79f20 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/a35fd9ed4dfe45bf997f43d535c79f20 2024-11-07T09:36:25,155 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/530961d9eb6e44bea45ef59833ce0166 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/530961d9eb6e44bea45ef59833ce0166 2024-11-07T09:36:25,156 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/2d90565175e24ee69163e3dad96b830c to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/2d90565175e24ee69163e3dad96b830c 2024-11-07T09:36:25,157 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/b4524744f63a47bcae0971726a59b256 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/b4524744f63a47bcae0971726a59b256 2024-11-07T09:36:25,159 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/8deb17a08c9b4dbf964ddb8d1c91207f to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/8deb17a08c9b4dbf964ddb8d1c91207f 2024-11-07T09:36:25,160 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/7fe787f0edbe4d8e887bcf8c9c13f61c to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/7fe787f0edbe4d8e887bcf8c9c13f61c 2024-11-07T09:36:25,161 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/57a9322168194069be335e259b9f9e26 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/57a9322168194069be335e259b9f9e26 2024-11-07T09:36:25,163 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/0f57baf32f7e4011837b287364dbc70d to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/0f57baf32f7e4011837b287364dbc70d 2024-11-07T09:36:25,164 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/d37be101ee83498fbd1ec23b340b06c4 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/d37be101ee83498fbd1ec23b340b06c4 2024-11-07T09:36:25,165 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/7de168851516462ea018c70ce97092c1 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/7de168851516462ea018c70ce97092c1 2024-11-07T09:36:25,184 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/fe938fc26f89444f9ea7d7b631d666ab, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/2ecd8476893642a193bf55c81cbb8f14, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/e8abba69d23e411dad3b49880a0378c5, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/76aa031fdcd34dd3949c9364ab4c5e49, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/1105290ccee740d1b7a0017563b6ffa4, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/d417061d29314c99946b5975d4b7d332, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/68b762e4c8bf43ef9f1546c676664336, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/2f4002f0bcd8458381618626574ecbe8, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/8feb7caf9df54c37b7a2c0ae10d72c7a, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/0379cd87cfca4b0bb7142ebda87b63c4, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/2e00e4f98372434aa2894da62117084f, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/baaba6cc6572451c9152cd9ad571b5b3, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/75b3fe5ae2d54d34838c56b7dca1eee4, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/a4bfbfe2c6434b55a7b956bda54efd1e, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/7c40733fb8f1444990b9dee089d14cfc, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/4aedce6aedcb488b93999dc006c440f0, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/6951fbd9c41c4a70b77f0c14961dd8bc, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/0c0a839cf4784be88a36b01af7ef459a, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/dffd9b43a7cd49f0936e35152fa012dd, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/310be4268321483caee35b189eb83a12, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/55618b88d39043f3b9a6fe65b34e9670, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/38afe9c89e6c42a88cb0925de7d21425, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/00aae78f067a4dc2813d7e7c256355fa, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/be320cbd5ad64be4b404667c8746bbd5, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/544f7a4e9dbc4f0b9a7bb2d93500253a, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/35b6ff9e469547a1a56be6d37d956276, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/6519dcaa9ec44f43a9ec9733c510cafc, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/ff24f01394d34b8d90b1c4126c793358, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/5cb23d5a5d3b4fc8952baaeb636938eb, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/068fd5c449dc4f70883e0bcbab28e6a7, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/3b696f4ae2ef4ec69d766bb51ec12119, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/86a433b552cb4a14ad2e91b998366890, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/df9bff6ddd5d4d3fabf3f98de3379b33, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/d578fe4e0ccd4736b7abe4bdd59eb148, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/289cb93affa54c7f8bc9255485ca8da0, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/01704f981b7b42a087dfcab4c156be1b, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/6d4e3442476e40d2af5935a77535cd26, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/d4c6400943d84010a07669eae3cae720] to archive 2024-11-07T09:36:25,185 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T09:36:25,187 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/fe938fc26f89444f9ea7d7b631d666ab to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/fe938fc26f89444f9ea7d7b631d666ab 2024-11-07T09:36:25,189 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/2ecd8476893642a193bf55c81cbb8f14 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/2ecd8476893642a193bf55c81cbb8f14 2024-11-07T09:36:25,190 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/e8abba69d23e411dad3b49880a0378c5 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/e8abba69d23e411dad3b49880a0378c5 2024-11-07T09:36:25,191 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/76aa031fdcd34dd3949c9364ab4c5e49 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/76aa031fdcd34dd3949c9364ab4c5e49 2024-11-07T09:36:25,193 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/1105290ccee740d1b7a0017563b6ffa4 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/1105290ccee740d1b7a0017563b6ffa4 2024-11-07T09:36:25,194 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/d417061d29314c99946b5975d4b7d332 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/d417061d29314c99946b5975d4b7d332 2024-11-07T09:36:25,196 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/68b762e4c8bf43ef9f1546c676664336 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/68b762e4c8bf43ef9f1546c676664336 2024-11-07T09:36:25,197 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/2f4002f0bcd8458381618626574ecbe8 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/2f4002f0bcd8458381618626574ecbe8 2024-11-07T09:36:25,198 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/8feb7caf9df54c37b7a2c0ae10d72c7a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/8feb7caf9df54c37b7a2c0ae10d72c7a 2024-11-07T09:36:25,199 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/0379cd87cfca4b0bb7142ebda87b63c4 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/0379cd87cfca4b0bb7142ebda87b63c4 2024-11-07T09:36:25,201 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/2e00e4f98372434aa2894da62117084f to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/2e00e4f98372434aa2894da62117084f 2024-11-07T09:36:25,202 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/baaba6cc6572451c9152cd9ad571b5b3 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/baaba6cc6572451c9152cd9ad571b5b3 2024-11-07T09:36:25,203 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/75b3fe5ae2d54d34838c56b7dca1eee4 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/75b3fe5ae2d54d34838c56b7dca1eee4 2024-11-07T09:36:25,205 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/a4bfbfe2c6434b55a7b956bda54efd1e to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/a4bfbfe2c6434b55a7b956bda54efd1e 2024-11-07T09:36:25,206 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/7c40733fb8f1444990b9dee089d14cfc to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/7c40733fb8f1444990b9dee089d14cfc 2024-11-07T09:36:25,208 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/4aedce6aedcb488b93999dc006c440f0 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/4aedce6aedcb488b93999dc006c440f0 2024-11-07T09:36:25,209 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/6951fbd9c41c4a70b77f0c14961dd8bc to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/6951fbd9c41c4a70b77f0c14961dd8bc 2024-11-07T09:36:25,210 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/0c0a839cf4784be88a36b01af7ef459a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/0c0a839cf4784be88a36b01af7ef459a 2024-11-07T09:36:25,212 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/dffd9b43a7cd49f0936e35152fa012dd to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/dffd9b43a7cd49f0936e35152fa012dd 2024-11-07T09:36:25,213 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/310be4268321483caee35b189eb83a12 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/310be4268321483caee35b189eb83a12 2024-11-07T09:36:25,214 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/55618b88d39043f3b9a6fe65b34e9670 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/55618b88d39043f3b9a6fe65b34e9670 2024-11-07T09:36:25,216 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/38afe9c89e6c42a88cb0925de7d21425 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/38afe9c89e6c42a88cb0925de7d21425 2024-11-07T09:36:25,217 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/00aae78f067a4dc2813d7e7c256355fa to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/00aae78f067a4dc2813d7e7c256355fa 2024-11-07T09:36:25,218 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/be320cbd5ad64be4b404667c8746bbd5 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/be320cbd5ad64be4b404667c8746bbd5 2024-11-07T09:36:25,219 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/544f7a4e9dbc4f0b9a7bb2d93500253a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/544f7a4e9dbc4f0b9a7bb2d93500253a 2024-11-07T09:36:25,221 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/35b6ff9e469547a1a56be6d37d956276 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/35b6ff9e469547a1a56be6d37d956276 2024-11-07T09:36:25,222 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/6519dcaa9ec44f43a9ec9733c510cafc to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/6519dcaa9ec44f43a9ec9733c510cafc 2024-11-07T09:36:25,223 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/ff24f01394d34b8d90b1c4126c793358 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/ff24f01394d34b8d90b1c4126c793358 2024-11-07T09:36:25,225 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/5cb23d5a5d3b4fc8952baaeb636938eb to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/5cb23d5a5d3b4fc8952baaeb636938eb 2024-11-07T09:36:25,226 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/068fd5c449dc4f70883e0bcbab28e6a7 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/068fd5c449dc4f70883e0bcbab28e6a7 2024-11-07T09:36:25,227 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/3b696f4ae2ef4ec69d766bb51ec12119 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/3b696f4ae2ef4ec69d766bb51ec12119 2024-11-07T09:36:25,228 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/86a433b552cb4a14ad2e91b998366890 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/86a433b552cb4a14ad2e91b998366890 2024-11-07T09:36:25,229 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/df9bff6ddd5d4d3fabf3f98de3379b33 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/df9bff6ddd5d4d3fabf3f98de3379b33 2024-11-07T09:36:25,231 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/d578fe4e0ccd4736b7abe4bdd59eb148 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/d578fe4e0ccd4736b7abe4bdd59eb148 2024-11-07T09:36:25,232 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/289cb93affa54c7f8bc9255485ca8da0 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/289cb93affa54c7f8bc9255485ca8da0 2024-11-07T09:36:25,233 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/01704f981b7b42a087dfcab4c156be1b to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/01704f981b7b42a087dfcab4c156be1b 2024-11-07T09:36:25,234 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/6d4e3442476e40d2af5935a77535cd26 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/6d4e3442476e40d2af5935a77535cd26 2024-11-07T09:36:25,236 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/d4c6400943d84010a07669eae3cae720 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/d4c6400943d84010a07669eae3cae720 2024-11-07T09:36:25,237 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/5b195c81a14742979ae62ae704dda98b, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/6e16b0752438417b8d1891ec389cfdf0, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/97f7de0297b14b82a283b221d4494165, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/eb0801396c75482781c6dbfe4207462d, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/39b6ac24adc7492bb4c77bce268e3405, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/7f8174412ac14cf2b29f1bd83b5ffa6e, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/3290cd1e23284841a76ae1ed8b27a4c5, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/57e9bf98e2c042c98ea9f31d8047168a, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/648a2109bc284ede837fa3fdae8bd8d9, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/dcfa1fbedc1f4e2a9e97d0f19cd6aa61, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/d16e972925ab4a41bb5b2532034a9a4b, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/5f519eb6cc6d433f8cb529c0bbc9bd06, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/d3330ca2c9be45c09b30e37286642983, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/a8b5e47306714288802dd6502b0a519d, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/49986d0a00324c30a09dc20bf790b14a, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/13972d40cb9e40b394430d9e41e334e5, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/970c7ecb9b64484585c148efd66bb851, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/5fe6ec9030bf4d419f307d59dbfdc007, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/f40511bd2dc24376ad090c47a8d5f353, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/0b307e79369644f3ad116943bb008516, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/ae192a51f92942568245bf9ab180c8ff, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/8a254dfe0e624ca3b2c757278daee4fd, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/0a324d7036dd4ed0851570f49ef5505d, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/f3337699030041d8a1f5d402e35a7506, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/17ec7b8ae36e462184a008c0637152cd, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/b0d44551c26d4a478aef1d6ee446b8a3, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/81a1a37cb4f941d881a4539ad3fb96eb, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/27b72e1e6ccc4ae2b3700ebdde578ea5, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/51a9242399aa42d0b82b5179ba0eb3c5, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/8d2099c6245741649815a0789c72d4dc, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/8e5afecf7a5b4ce5814b702430157e39, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/4874583fdc6c4b16960ab6422ae1fdfe, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/0b9135cefaa845b2bbe79f6a388f71b5, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/7001642959754d16b89ab32d65ad1590, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/96240a0f86ac4d9aa072f0825b748543, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/3b7c2318507f4f00bcdd17ee40d622ba, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/2f29f1ea4b6e4562bddf81bd92a26554, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/8d3ec6a1e574497c8aba11cf592c679c] to archive 2024-11-07T09:36:25,238 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T09:36:25,240 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/5b195c81a14742979ae62ae704dda98b to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/5b195c81a14742979ae62ae704dda98b 2024-11-07T09:36:25,241 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/6e16b0752438417b8d1891ec389cfdf0 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/6e16b0752438417b8d1891ec389cfdf0 2024-11-07T09:36:25,243 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/97f7de0297b14b82a283b221d4494165 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/97f7de0297b14b82a283b221d4494165 2024-11-07T09:36:25,244 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/eb0801396c75482781c6dbfe4207462d to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/eb0801396c75482781c6dbfe4207462d 2024-11-07T09:36:25,245 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/39b6ac24adc7492bb4c77bce268e3405 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/39b6ac24adc7492bb4c77bce268e3405 2024-11-07T09:36:25,247 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/7f8174412ac14cf2b29f1bd83b5ffa6e to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/7f8174412ac14cf2b29f1bd83b5ffa6e 2024-11-07T09:36:25,248 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/3290cd1e23284841a76ae1ed8b27a4c5 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/3290cd1e23284841a76ae1ed8b27a4c5 2024-11-07T09:36:25,249 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/57e9bf98e2c042c98ea9f31d8047168a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/57e9bf98e2c042c98ea9f31d8047168a 2024-11-07T09:36:25,250 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/648a2109bc284ede837fa3fdae8bd8d9 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/648a2109bc284ede837fa3fdae8bd8d9 2024-11-07T09:36:25,252 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/dcfa1fbedc1f4e2a9e97d0f19cd6aa61 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/dcfa1fbedc1f4e2a9e97d0f19cd6aa61 2024-11-07T09:36:25,253 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/d16e972925ab4a41bb5b2532034a9a4b to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/d16e972925ab4a41bb5b2532034a9a4b 2024-11-07T09:36:25,254 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/5f519eb6cc6d433f8cb529c0bbc9bd06 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/5f519eb6cc6d433f8cb529c0bbc9bd06 2024-11-07T09:36:25,256 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/d3330ca2c9be45c09b30e37286642983 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/d3330ca2c9be45c09b30e37286642983 2024-11-07T09:36:25,257 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/a8b5e47306714288802dd6502b0a519d to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/a8b5e47306714288802dd6502b0a519d 2024-11-07T09:36:25,258 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/49986d0a00324c30a09dc20bf790b14a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/49986d0a00324c30a09dc20bf790b14a 2024-11-07T09:36:25,259 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/13972d40cb9e40b394430d9e41e334e5 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/13972d40cb9e40b394430d9e41e334e5 2024-11-07T09:36:25,261 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/970c7ecb9b64484585c148efd66bb851 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/970c7ecb9b64484585c148efd66bb851 2024-11-07T09:36:25,262 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/5fe6ec9030bf4d419f307d59dbfdc007 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/5fe6ec9030bf4d419f307d59dbfdc007 2024-11-07T09:36:25,263 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/f40511bd2dc24376ad090c47a8d5f353 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/f40511bd2dc24376ad090c47a8d5f353 2024-11-07T09:36:25,264 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/0b307e79369644f3ad116943bb008516 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/0b307e79369644f3ad116943bb008516 2024-11-07T09:36:25,265 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/ae192a51f92942568245bf9ab180c8ff to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/ae192a51f92942568245bf9ab180c8ff 2024-11-07T09:36:25,267 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/8a254dfe0e624ca3b2c757278daee4fd to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/8a254dfe0e624ca3b2c757278daee4fd 2024-11-07T09:36:25,268 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/0a324d7036dd4ed0851570f49ef5505d to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/0a324d7036dd4ed0851570f49ef5505d 2024-11-07T09:36:25,269 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/f3337699030041d8a1f5d402e35a7506 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/f3337699030041d8a1f5d402e35a7506 2024-11-07T09:36:25,270 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/17ec7b8ae36e462184a008c0637152cd to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/17ec7b8ae36e462184a008c0637152cd 2024-11-07T09:36:25,271 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/b0d44551c26d4a478aef1d6ee446b8a3 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/b0d44551c26d4a478aef1d6ee446b8a3 2024-11-07T09:36:25,272 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/81a1a37cb4f941d881a4539ad3fb96eb to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/81a1a37cb4f941d881a4539ad3fb96eb 2024-11-07T09:36:25,273 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/27b72e1e6ccc4ae2b3700ebdde578ea5 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/27b72e1e6ccc4ae2b3700ebdde578ea5 2024-11-07T09:36:25,274 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/51a9242399aa42d0b82b5179ba0eb3c5 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/51a9242399aa42d0b82b5179ba0eb3c5 2024-11-07T09:36:25,276 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/8d2099c6245741649815a0789c72d4dc to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/8d2099c6245741649815a0789c72d4dc 2024-11-07T09:36:25,277 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/8e5afecf7a5b4ce5814b702430157e39 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/8e5afecf7a5b4ce5814b702430157e39 2024-11-07T09:36:25,278 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/4874583fdc6c4b16960ab6422ae1fdfe to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/4874583fdc6c4b16960ab6422ae1fdfe 2024-11-07T09:36:25,279 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/0b9135cefaa845b2bbe79f6a388f71b5 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/0b9135cefaa845b2bbe79f6a388f71b5 2024-11-07T09:36:25,280 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/7001642959754d16b89ab32d65ad1590 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/7001642959754d16b89ab32d65ad1590 2024-11-07T09:36:25,281 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/96240a0f86ac4d9aa072f0825b748543 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/96240a0f86ac4d9aa072f0825b748543 2024-11-07T09:36:25,283 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/3b7c2318507f4f00bcdd17ee40d622ba to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/3b7c2318507f4f00bcdd17ee40d622ba 2024-11-07T09:36:25,284 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/2f29f1ea4b6e4562bddf81bd92a26554 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/2f29f1ea4b6e4562bddf81bd92a26554 2024-11-07T09:36:25,285 DEBUG [StoreCloser-TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/8d3ec6a1e574497c8aba11cf592c679c to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/8d3ec6a1e574497c8aba11cf592c679c 2024-11-07T09:36:25,290 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/recovered.edits/575.seqid, newMaxSeqId=575, maxSeqId=1 2024-11-07T09:36:25,291 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. 2024-11-07T09:36:25,291 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1676): Region close journal for 116dccc5098610cd38ab507cb3daad26: Waiting for close lock at 1730972183856Running coprocessor pre-close hooks at 1730972183856Disabling compacts and flushes for region at 1730972183856Disabling writes for close at 1730972183857 (+1 ms)Obtaining lock to block concurrent updates at 1730972183857Preparing flush snapshotting stores in 116dccc5098610cd38ab507cb3daad26 at 1730972183857Finished memstore snapshotting TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=36720, getOffHeapSize=0, getCellsCount=300 at 1730972183857Flushing stores of TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26. at 1730972183858 (+1 ms)Flushing 116dccc5098610cd38ab507cb3daad26/A: creating writer at 1730972183858Flushing 116dccc5098610cd38ab507cb3daad26/A: appending metadata at 1730972183861 (+3 ms)Flushing 116dccc5098610cd38ab507cb3daad26/A: closing flushed file at 1730972183861Flushing 116dccc5098610cd38ab507cb3daad26/B: creating writer at 1730972184271 (+410 ms)Flushing 116dccc5098610cd38ab507cb3daad26/B: appending metadata at 1730972184273 (+2 ms)Flushing 116dccc5098610cd38ab507cb3daad26/B: closing flushed file at 1730972184273Flushing 116dccc5098610cd38ab507cb3daad26/C: creating writer at 1730972184683 (+410 ms)Flushing 116dccc5098610cd38ab507cb3daad26/C: appending metadata at 1730972184685 (+2 ms)Flushing 116dccc5098610cd38ab507cb3daad26/C: closing flushed file at 1730972184685Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c334e60: reopening flushed file at 1730972185095 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@40ed6d55: reopening flushed file at 1730972185101 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@103e4e08: reopening flushed file at 1730972185106 (+5 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 116dccc5098610cd38ab507cb3daad26 in 1254ms, sequenceid=572, compaction requested=true at 1730972185111 (+5 ms)Writing region close event to WAL at 1730972185286 (+175 ms)Running coprocessor post-close hooks at 1730972185291 (+5 ms)Closed at 1730972185291 2024-11-07T09:36:25,294 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(157): Closed 116dccc5098610cd38ab507cb3daad26 2024-11-07T09:36:25,295 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=116dccc5098610cd38ab507cb3daad26, regionState=CLOSED 2024-11-07T09:36:25,296 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 because future has completed 2024-11-07T09:36:25,299 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-07T09:36:25,299 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 116dccc5098610cd38ab507cb3daad26, server=9f1eb259a29b,42895,1730972153970 in 1.5990 sec 2024-11-07T09:36:25,301 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-07T09:36:25,301 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=116dccc5098610cd38ab507cb3daad26, UNASSIGN in 1.6050 sec 2024-11-07T09:36:25,304 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-07T09:36:25,304 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6120 sec 2024-11-07T09:36:25,305 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730972185305"}]},"ts":"1730972185305"} 2024-11-07T09:36:25,307 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-07T09:36:25,307 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-07T09:36:25,309 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6250 sec 2024-11-07T09:36:25,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-07T09:36:25,833 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:36:25,836 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-07T09:36:25,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T09:36:25,842 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=35, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T09:36:25,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-07T09:36:25,843 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=35, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T09:36:25,848 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26 2024-11-07T09:36:25,851 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A, FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B, FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C, FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/recovered.edits] 2024-11-07T09:36:25,855 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/1fd111c99880444287bad7cc0214afae to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/1fd111c99880444287bad7cc0214afae 2024-11-07T09:36:25,856 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/df599fbc21e142de8c670c60920dec23 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/df599fbc21e142de8c670c60920dec23 2024-11-07T09:36:25,857 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/eb09b9e33b2c447db390473f1c1aa3fa to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/A/eb09b9e33b2c447db390473f1c1aa3fa 2024-11-07T09:36:25,860 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/1c8f48077ced43159e2643d987dfbaf0 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/1c8f48077ced43159e2643d987dfbaf0 2024-11-07T09:36:25,861 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/74a7adf09e5c4262aa893411cbd3e48a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/74a7adf09e5c4262aa893411cbd3e48a 2024-11-07T09:36:25,863 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/f43ab72b640246fda5c15d588c2eebb2 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/B/f43ab72b640246fda5c15d588c2eebb2 2024-11-07T09:36:25,866 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/2170211cde6e43e9b2b4acbc0d2c90c5 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/2170211cde6e43e9b2b4acbc0d2c90c5 2024-11-07T09:36:25,867 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/3028693de8a84d76ad79e08d2abbb4a1 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/3028693de8a84d76ad79e08d2abbb4a1 2024-11-07T09:36:25,869 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/856bcbc0e30a40f1ae5c1dc86e81d4ee to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/C/856bcbc0e30a40f1ae5c1dc86e81d4ee 2024-11-07T09:36:25,872 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/recovered.edits/575.seqid to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26/recovered.edits/575.seqid 2024-11-07T09:36:25,873 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/116dccc5098610cd38ab507cb3daad26 2024-11-07T09:36:25,873 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-07T09:36:25,878 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=35, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T09:36:25,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-07T09:36:25,888 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-07T09:36:25,923 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-07T09:36:25,925 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=35, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T09:36:25,925 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-07T09:36:25,926 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1730972185925"}]},"ts":"9223372036854775807"} 2024-11-07T09:36:25,930 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-07T09:36:25,930 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 116dccc5098610cd38ab507cb3daad26, NAME => 'TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26.', STARTKEY => '', ENDKEY => ''}] 2024-11-07T09:36:25,931 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-07T09:36:25,931 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1730972185931"}]},"ts":"9223372036854775807"} 2024-11-07T09:36:25,934 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-07T09:36:25,935 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=35, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T09:36:25,937 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 98 msec 2024-11-07T09:36:25,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-07T09:36:25,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-07T09:36:25,954 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:36:25,967 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=238 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2090188048_22 at /127.0.0.1:37530 [Waiting for operation #375] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;9f1eb259a29b:42895-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_651746630_22 at /127.0.0.1:53776 [Waiting for operation #388] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=391 (was 315) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4899 (was 5483) 2024-11-07T09:36:25,976 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=238, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=391, ProcessCount=11, AvailableMemoryMB=4898 2024-11-07T09:36:25,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T09:36:25,978 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T09:36:25,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-07T09:36:25,980 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T09:36:25,980 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:25,980 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 36 2024-11-07T09:36:25,981 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T09:36:25,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-07T09:36:25,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741960_1136 (size=959) 2024-11-07T09:36:26,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-07T09:36:26,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-07T09:36:26,390 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260 2024-11-07T09:36:26,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741961_1137 (size=53) 2024-11-07T09:36:26,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-07T09:36:26,796 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T09:36:26,796 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 81b216f4bb32d787c444da8261b91891, disabling compactions & flushes 2024-11-07T09:36:26,796 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:26,796 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:26,796 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. after waiting 0 ms 2024-11-07T09:36:26,796 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:26,796 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:26,796 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 81b216f4bb32d787c444da8261b91891: Waiting for close lock at 1730972186796Disabling compacts and flushes for region at 1730972186796Disabling writes for close at 1730972186796Writing region close event to WAL at 1730972186796Closed at 1730972186796 2024-11-07T09:36:26,798 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T09:36:26,798 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1730972186798"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730972186798"}]},"ts":"1730972186798"} 2024-11-07T09:36:26,800 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-07T09:36:26,801 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T09:36:26,801 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730972186801"}]},"ts":"1730972186801"} 2024-11-07T09:36:26,802 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-07T09:36:26,803 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=81b216f4bb32d787c444da8261b91891, ASSIGN}] 2024-11-07T09:36:26,804 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=81b216f4bb32d787c444da8261b91891, ASSIGN 2024-11-07T09:36:26,805 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=81b216f4bb32d787c444da8261b91891, ASSIGN; state=OFFLINE, location=9f1eb259a29b,42895,1730972153970; forceNewPlan=false, retain=false 2024-11-07T09:36:26,955 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=81b216f4bb32d787c444da8261b91891, regionState=OPENING, regionLocation=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:26,957 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=81b216f4bb32d787c444da8261b91891, ASSIGN because future has completed 2024-11-07T09:36:26,958 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970}] 2024-11-07T09:36:27,113 INFO [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:27,114 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} 2024-11-07T09:36:27,114 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:27,114 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T09:36:27,114 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for 81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:27,114 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for 81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:27,116 INFO [StoreOpener-81b216f4bb32d787c444da8261b91891-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:27,118 INFO [StoreOpener-81b216f4bb32d787c444da8261b91891-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81b216f4bb32d787c444da8261b91891 columnFamilyName A 2024-11-07T09:36:27,118 DEBUG [StoreOpener-81b216f4bb32d787c444da8261b91891-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:27,118 INFO [StoreOpener-81b216f4bb32d787c444da8261b91891-1 {}] regionserver.HStore(327): Store=81b216f4bb32d787c444da8261b91891/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T09:36:27,119 INFO [StoreOpener-81b216f4bb32d787c444da8261b91891-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:27,120 INFO [StoreOpener-81b216f4bb32d787c444da8261b91891-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81b216f4bb32d787c444da8261b91891 columnFamilyName B 2024-11-07T09:36:27,120 DEBUG [StoreOpener-81b216f4bb32d787c444da8261b91891-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:27,120 INFO [StoreOpener-81b216f4bb32d787c444da8261b91891-1 {}] regionserver.HStore(327): Store=81b216f4bb32d787c444da8261b91891/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T09:36:27,120 INFO [StoreOpener-81b216f4bb32d787c444da8261b91891-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:27,121 INFO [StoreOpener-81b216f4bb32d787c444da8261b91891-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81b216f4bb32d787c444da8261b91891 columnFamilyName C 2024-11-07T09:36:27,121 DEBUG [StoreOpener-81b216f4bb32d787c444da8261b91891-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:27,122 INFO [StoreOpener-81b216f4bb32d787c444da8261b91891-1 {}] regionserver.HStore(327): Store=81b216f4bb32d787c444da8261b91891/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T09:36:27,122 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for 81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:27,123 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:27,123 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-07T09:36:27,124 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for 81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:27,124 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for 81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:27,125 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T09:36:27,126 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for 81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:27,127 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T09:36:27,128 INFO [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened 81b216f4bb32d787c444da8261b91891; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71184345, jitterRate=0.06072939932346344}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T09:36:27,128 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:27,129 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for 81b216f4bb32d787c444da8261b91891: Running coprocessor pre-open hook at 1730972187114Writing region info on filesystem at 1730972187114Initializing all the Stores at 1730972187115 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730972187115Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730972187115Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730972187115Cleaning up temporary data from old regions at 1730972187124 (+9 ms)Running coprocessor post-open hooks at 1730972187128 (+4 ms)Region opened successfully at 1730972187129 (+1 ms) 2024-11-07T09:36:27,130 INFO [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., pid=38, masterSystemTime=1730972187110 2024-11-07T09:36:27,132 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:27,132 INFO [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:27,132 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=81b216f4bb32d787c444da8261b91891, regionState=OPEN, openSeqNum=2, regionLocation=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:27,134 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 because future has completed 2024-11-07T09:36:27,136 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-07T09:36:27,136 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure 81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 in 177 msec 2024-11-07T09:36:27,138 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-07T09:36:27,138 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=81b216f4bb32d787c444da8261b91891, ASSIGN in 333 msec 2024-11-07T09:36:27,139 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T09:36:27,139 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730972187139"}]},"ts":"1730972187139"} 2024-11-07T09:36:27,140 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-07T09:36:27,141 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T09:36:27,143 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1630 sec 2024-11-07T09:36:28,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-07T09:36:28,133 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:36:28,134 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@769bdb6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:28,134 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:36:28,134 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:36:28,136 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:36:28,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:36:28,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:36:28,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66120160, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:28,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:36:28,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:36:28,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:28,138 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35086, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:36:28,139 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bde401d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:28,139 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:36:28,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:36:28,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:36:28,142 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37556, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:36:28,143 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T09:36:28,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 9f1eb259a29b,41689,1730972153310 2024-11-07T09:36:28,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@55c63cb3 2024-11-07T09:36:28,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T09:36:28,145 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35098, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T09:36:28,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T09:36:28,150 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T09:36:28,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=39, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-07T09:36:28,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741962_1138 (size=995) 2024-11-07T09:36:28,569 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-07T09:36:28,569 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-07T09:36:28,573 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T09:36:28,580 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=81b216f4bb32d787c444da8261b91891, REOPEN/MOVE}] 2024-11-07T09:36:28,581 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=81b216f4bb32d787c444da8261b91891, REOPEN/MOVE 2024-11-07T09:36:28,582 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=81b216f4bb32d787c444da8261b91891, regionState=CLOSING, regionLocation=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:28,583 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=81b216f4bb32d787c444da8261b91891, REOPEN/MOVE because future has completed 2024-11-07T09:36:28,584 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T09:36:28,584 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=41, state=RUNNABLE, hasLock=false; CloseRegionProcedure 81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970}] 2024-11-07T09:36:28,736 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] handler.UnassignRegionHandler(122): Close 81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:28,736 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T09:36:28,736 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1722): Closing 81b216f4bb32d787c444da8261b91891, disabling compactions & flushes 2024-11-07T09:36:28,737 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:28,737 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:28,737 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. after waiting 0 ms 2024-11-07T09:36:28,737 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:28,741 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T09:36:28,741 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:28,741 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1676): Region close journal for 81b216f4bb32d787c444da8261b91891: Waiting for close lock at 1730972188736Running coprocessor pre-close hooks at 1730972188736Disabling compacts and flushes for region at 1730972188736Disabling writes for close at 1730972188737 (+1 ms)Writing region close event to WAL at 1730972188737Running coprocessor post-close hooks at 1730972188741 (+4 ms)Closed at 1730972188741 2024-11-07T09:36:28,741 WARN [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegionServer(3299): Not adding moved region record: 81b216f4bb32d787c444da8261b91891 to self. 2024-11-07T09:36:28,743 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] handler.UnassignRegionHandler(157): Closed 81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:28,744 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=81b216f4bb32d787c444da8261b91891, regionState=CLOSED 2024-11-07T09:36:28,746 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=41, state=RUNNABLE, hasLock=false; CloseRegionProcedure 81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 because future has completed 2024-11-07T09:36:28,748 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=41 2024-11-07T09:36:28,748 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=41, state=SUCCESS, hasLock=false; CloseRegionProcedure 81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 in 162 msec 2024-11-07T09:36:28,749 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=81b216f4bb32d787c444da8261b91891, REOPEN/MOVE; state=CLOSED, location=9f1eb259a29b,42895,1730972153970; forceNewPlan=false, retain=true 2024-11-07T09:36:28,899 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=81b216f4bb32d787c444da8261b91891, regionState=OPENING, regionLocation=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:28,901 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=81b216f4bb32d787c444da8261b91891, REOPEN/MOVE because future has completed 2024-11-07T09:36:28,901 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=41, state=RUNNABLE, hasLock=false; OpenRegionProcedure 81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970}] 2024-11-07T09:36:29,057 INFO [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:29,057 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} 2024-11-07T09:36:29,057 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:29,057 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T09:36:29,057 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for 81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:29,057 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for 81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:29,062 INFO [StoreOpener-81b216f4bb32d787c444da8261b91891-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:29,067 INFO [StoreOpener-81b216f4bb32d787c444da8261b91891-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81b216f4bb32d787c444da8261b91891 columnFamilyName A 2024-11-07T09:36:29,069 DEBUG [StoreOpener-81b216f4bb32d787c444da8261b91891-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:29,069 INFO [StoreOpener-81b216f4bb32d787c444da8261b91891-1 {}] regionserver.HStore(327): Store=81b216f4bb32d787c444da8261b91891/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T09:36:29,070 INFO [StoreOpener-81b216f4bb32d787c444da8261b91891-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:29,070 INFO [StoreOpener-81b216f4bb32d787c444da8261b91891-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81b216f4bb32d787c444da8261b91891 columnFamilyName B 2024-11-07T09:36:29,071 DEBUG [StoreOpener-81b216f4bb32d787c444da8261b91891-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:29,071 INFO [StoreOpener-81b216f4bb32d787c444da8261b91891-1 {}] regionserver.HStore(327): Store=81b216f4bb32d787c444da8261b91891/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T09:36:29,071 INFO [StoreOpener-81b216f4bb32d787c444da8261b91891-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:29,072 INFO [StoreOpener-81b216f4bb32d787c444da8261b91891-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81b216f4bb32d787c444da8261b91891 columnFamilyName C 2024-11-07T09:36:29,072 DEBUG [StoreOpener-81b216f4bb32d787c444da8261b91891-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:29,072 INFO [StoreOpener-81b216f4bb32d787c444da8261b91891-1 {}] regionserver.HStore(327): Store=81b216f4bb32d787c444da8261b91891/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T09:36:29,072 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for 81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:29,073 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:29,074 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:29,075 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for 81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:29,075 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for 81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:29,076 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T09:36:29,078 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for 81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:29,079 INFO [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened 81b216f4bb32d787c444da8261b91891; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73213890, jitterRate=0.09097197651863098}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T09:36:29,079 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:29,080 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for 81b216f4bb32d787c444da8261b91891: Running coprocessor pre-open hook at 1730972189058Writing region info on filesystem at 1730972189058Initializing all the Stores at 1730972189058Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730972189058Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730972189059 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730972189059Cleaning up temporary data from old regions at 1730972189075 (+16 ms)Running coprocessor post-open hooks at 1730972189079 (+4 ms)Region opened successfully at 1730972189080 (+1 ms) 2024-11-07T09:36:29,081 INFO [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., pid=43, masterSystemTime=1730972189053 2024-11-07T09:36:29,083 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:29,083 INFO [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:29,084 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=81b216f4bb32d787c444da8261b91891, regionState=OPEN, openSeqNum=5, regionLocation=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:29,086 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=41, state=RUNNABLE, hasLock=false; OpenRegionProcedure 81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 because future has completed 2024-11-07T09:36:29,088 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=41 2024-11-07T09:36:29,089 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=41, state=SUCCESS, hasLock=false; OpenRegionProcedure 81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 in 186 msec 2024-11-07T09:36:29,091 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-07T09:36:29,091 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=81b216f4bb32d787c444da8261b91891, REOPEN/MOVE in 509 msec 2024-11-07T09:36:29,094 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-07T09:36:29,094 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 519 msec 2024-11-07T09:36:29,098 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 943 msec 2024-11-07T09:36:29,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-07T09:36:29,100 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:36:29,100 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18f01ca4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:29,101 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:36:29,101 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:36:29,102 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:36:29,102 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:36:29,102 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:36:29,102 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@542bfdb2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:29,102 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:36:29,102 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:36:29,103 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:29,103 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35116, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:36:29,104 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2df94266, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:29,105 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a283555, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:29,105 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:36:29,105 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:36:29,106 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:36:29,106 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:36:29,106 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:36:29,106 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@468dad96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:29,106 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:36:29,107 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:36:29,107 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:29,108 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35138, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:36:29,108 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26cfddfe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:29,109 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ba7f2f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:29,109 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:36:29,109 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:36:29,110 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:36:29,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:36:29,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:36:29,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@758cd8ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:29,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:36:29,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:36:29,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:29,111 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35164, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:36:29,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c30cd3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:29,112 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14984fac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:29,112 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:36:29,112 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:36:29,113 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:36:29,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:36:29,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:36:29,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e00d919, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:29,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:36:29,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:36:29,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:29,114 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35188, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:36:29,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67000ea5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:29,115 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e890ec6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:29,115 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:36:29,116 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:36:29,116 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:36:29,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:36:29,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:36:29,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4344511a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:29,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:36:29,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:36:29,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:29,118 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35206, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:36:29,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@91a8fa2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:29,119 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f62ef44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:29,119 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:36:29,119 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:36:29,120 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:36:29,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:36:29,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:36:29,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b4b9b25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:29,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:36:29,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:36:29,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:29,122 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35218, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:36:29,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c069d2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:29,123 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f60d5da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:29,123 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:36:29,123 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:36:29,124 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:36:29,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:36:29,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:36:29,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@598a1cc5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:29,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:36:29,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:36:29,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:29,125 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35236, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:36:29,126 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47efecfa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:29,127 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@139f617f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:29,127 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:36:29,127 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:36:29,127 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:36:29,127 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:36:29,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:36:29,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@467b7819, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:29,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:36:29,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:36:29,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:29,129 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35254, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:36:29,129 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ea03db0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:29,130 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7def024f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:29,130 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:36:29,130 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:36:29,131 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:36:29,131 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:36:29,131 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:36:29,131 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b934d13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:29,131 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:36:29,131 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:36:29,132 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:29,132 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35278, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:36:29,133 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56a3a672, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:36:29,134 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T09:36:29,134 DEBUG [Thread-684 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:36:29,134 DEBUG [Thread-697 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:36:29,134 DEBUG [Thread-682 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:36:29,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:36:29,135 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:36:29,135 DEBUG [Thread-686 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:36:29,135 DEBUG [Thread-690 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:36:29,135 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:36:29,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:36:29,135 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:36:29,135 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:36:29,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-07T09:36:29,136 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37576, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:36:29,136 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37588, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:36:29,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:36:29,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:36:29,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-07T09:36:29,138 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37602, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:36:29,138 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T09:36:29,139 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T09:36:29,139 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T09:36:29,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5] 2024-11-07T09:36:29,143 DEBUG [Thread-695 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:36:29,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5] 2024-11-07T09:36:29,144 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5] 2024-11-07T09:36:29,144 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37570, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:36:29,144 DEBUG [Thread-690 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:36:29,145 DEBUG [Thread-695 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:36:29,145 DEBUG [Thread-695 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:36:29,146 DEBUG [Thread-690 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:36:29,147 DEBUG [Thread-699 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:36:29,147 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37606, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:36:29,147 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5] 2024-11-07T09:36:29,148 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37622, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:36:29,148 DEBUG [Thread-688 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:36:29,148 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:36:29,148 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:36:29,149 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37632, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:36:29,149 DEBUG [Thread-693 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:36:29,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:36:29,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:36:29,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:36:29,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:36:29,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5] 2024-11-07T09:36:29,151 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37640, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:36:29,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5] 2024-11-07T09:36:29,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5] 2024-11-07T09:36:29,161 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37648, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:36:29,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5] 2024-11-07T09:36:29,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5] 2024-11-07T09:36:29,171 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:36:29,171 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81b216f4bb32d787c444da8261b91891 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-07T09:36:29,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:29,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:29,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972249231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:29,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:29,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972249230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:29,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:29,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:29,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:29,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:29,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:29,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:29,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:29,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1730972249233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:29,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:29,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:29,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:29,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-07T09:36:29,252 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110710592d18cba24af1a23fea1fe1a31512_81b216f4bb32d787c444da8261b91891 is 50, key is test_row_0/A:col10/1730972189165/Put/seqid=0 2024-11-07T09:36:29,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:29,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972249239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:29,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:29,254 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:29,255 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:29,255 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:29,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972249239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:29,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:29,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:29,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:29,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741963_1139 (size=17034) 2024-11-07T09:36:29,288 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:29,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-07T09:36:29,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:29,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. as already flushing 2024-11-07T09:36:29,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:29,294 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:29,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:29,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:29,299 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110710592d18cba24af1a23fea1fe1a31512_81b216f4bb32d787c444da8261b91891 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110710592d18cba24af1a23fea1fe1a31512_81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:29,301 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/f41795ffda244dceb7871ee1d6f14c41, store: [table=TestAcidGuarantees family=A region=81b216f4bb32d787c444da8261b91891] 2024-11-07T09:36:29,309 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/f41795ffda244dceb7871ee1d6f14c41 is 175, key is test_row_0/A:col10/1730972189165/Put/seqid=0 2024-11-07T09:36:29,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741964_1140 (size=48135) 2024-11-07T09:36:29,335 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/f41795ffda244dceb7871ee1d6f14c41 2024-11-07T09:36:29,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:29,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972249343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:29,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:29,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:29,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:29,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:29,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972249343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:29,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:29,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:29,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:29,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:29,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1730972249343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:29,348 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:29,348 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:29,348 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:29,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:29,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:29,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972249363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:29,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972249363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:29,368 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:29,368 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:29,368 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:29,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:29,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:29,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:29,369 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/5d5aea263ee84112bb451e1fcbb5ea18 is 50, key is test_row_0/B:col10/1730972189165/Put/seqid=0 2024-11-07T09:36:29,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741965_1141 (size=9657) 2024-11-07T09:36:29,430 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/5d5aea263ee84112bb451e1fcbb5ea18 2024-11-07T09:36:29,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-07T09:36:29,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:29,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. as already flushing 2024-11-07T09:36:29,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:29,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:29,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:29,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:29,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-07T09:36:29,469 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/956d40ed423c4871aad9f07422e6709e is 50, key is test_row_0/C:col10/1730972189165/Put/seqid=0 2024-11-07T09:36:29,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741966_1142 (size=9657) 2024-11-07T09:36:29,498 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/956d40ed423c4871aad9f07422e6709e 2024-11-07T09:36:29,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/f41795ffda244dceb7871ee1d6f14c41 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/f41795ffda244dceb7871ee1d6f14c41 2024-11-07T09:36:29,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/f41795ffda244dceb7871ee1d6f14c41, entries=250, sequenceid=17, filesize=47.0 K 2024-11-07T09:36:29,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/5d5aea263ee84112bb451e1fcbb5ea18 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/5d5aea263ee84112bb451e1fcbb5ea18 2024-11-07T09:36:29,532 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/5d5aea263ee84112bb451e1fcbb5ea18, entries=100, sequenceid=17, filesize=9.4 K 2024-11-07T09:36:29,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/956d40ed423c4871aad9f07422e6709e as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/956d40ed423c4871aad9f07422e6709e 2024-11-07T09:36:29,540 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/956d40ed423c4871aad9f07422e6709e, entries=100, sequenceid=17, filesize=9.4 K 2024-11-07T09:36:29,541 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 81b216f4bb32d787c444da8261b91891 in 370ms, sequenceid=17, compaction requested=false 2024-11-07T09:36:29,541 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-07T09:36:29,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81b216f4bb32d787c444da8261b91891: 2024-11-07T09:36:29,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:29,557 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:36:29,557 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81b216f4bb32d787c444da8261b91891 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T09:36:29,581 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107455d4edc4dfb45418d83b14402255612_81b216f4bb32d787c444da8261b91891 is 50, key is test_row_0/A:col10/1730972189236/Put/seqid=0 2024-11-07T09:36:29,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:29,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1730972249574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:29,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:29,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972249576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:29,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:29,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:29,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:29,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:29,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:29,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:29,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:29,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972249585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:29,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:29,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:29,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:29,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:29,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972249587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:29,597 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:29,597 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:29,597 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:29,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741967_1143 (size=14594) 2024-11-07T09:36:29,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-07T09:36:29,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:29,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. as already flushing 2024-11-07T09:36:29,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:29,604 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:29,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:29,605 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:29,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:29,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972249590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:29,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:29,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:29,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:29,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:29,611 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107455d4edc4dfb45418d83b14402255612_81b216f4bb32d787c444da8261b91891 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107455d4edc4dfb45418d83b14402255612_81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:29,612 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/287bfdbe7ce7446c97d55f3bcc8cdbf2, store: [table=TestAcidGuarantees family=A region=81b216f4bb32d787c444da8261b91891] 2024-11-07T09:36:29,613 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/287bfdbe7ce7446c97d55f3bcc8cdbf2 is 175, key is test_row_0/A:col10/1730972189236/Put/seqid=0 2024-11-07T09:36:29,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741968_1144 (size=39549) 2024-11-07T09:36:29,643 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=42, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/287bfdbe7ce7446c97d55f3bcc8cdbf2 2024-11-07T09:36:29,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/0ca5866ee0d34230b06b405614e2ef45 is 50, key is test_row_0/B:col10/1730972189236/Put/seqid=0 2024-11-07T09:36:29,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741969_1145 (size=12001) 2024-11-07T09:36:29,675 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/0ca5866ee0d34230b06b405614e2ef45 2024-11-07T09:36:29,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/c2bc84f0e2b34f20848921d0a69d3487 is 50, key is test_row_0/C:col10/1730972189236/Put/seqid=0 2024-11-07T09:36:29,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:29,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1730972249693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:29,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:29,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:29,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:29,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:29,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:29,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972249693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:29,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972249693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:29,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:29,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:29,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:29,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:29,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:29,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:29,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:29,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972249703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:29,706 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:29,706 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:29,706 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:29,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:29,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972249714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:29,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:29,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:29,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:29,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741970_1146 (size=12001) 2024-11-07T09:36:29,722 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/c2bc84f0e2b34f20848921d0a69d3487 2024-11-07T09:36:29,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/287bfdbe7ce7446c97d55f3bcc8cdbf2 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/287bfdbe7ce7446c97d55f3bcc8cdbf2 2024-11-07T09:36:29,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/287bfdbe7ce7446c97d55f3bcc8cdbf2, entries=200, sequenceid=42, filesize=38.6 K 2024-11-07T09:36:29,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/0ca5866ee0d34230b06b405614e2ef45 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/0ca5866ee0d34230b06b405614e2ef45 2024-11-07T09:36:29,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/0ca5866ee0d34230b06b405614e2ef45, entries=150, sequenceid=42, filesize=11.7 K 2024-11-07T09:36:29,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/c2bc84f0e2b34f20848921d0a69d3487 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/c2bc84f0e2b34f20848921d0a69d3487 2024-11-07T09:36:29,751 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/c2bc84f0e2b34f20848921d0a69d3487, entries=150, sequenceid=42, filesize=11.7 K 2024-11-07T09:36:29,752 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 81b216f4bb32d787c444da8261b91891 in 195ms, sequenceid=42, compaction requested=false 2024-11-07T09:36:29,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81b216f4bb32d787c444da8261b91891: 2024-11-07T09:36:29,758 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-07T09:36:29,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:29,760 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 81b216f4bb32d787c444da8261b91891 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-07T09:36:29,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-07T09:36:29,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107da66a09f3b2349de9d1dee53bb530780_81b216f4bb32d787c444da8261b91891 is 50, key is test_row_0/A:col10/1730972189585/Put/seqid=0 2024-11-07T09:36:29,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741971_1147 (size=12154) 2024-11-07T09:36:29,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:29,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. as already flushing 2024-11-07T09:36:29,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:29,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972249942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:29,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:29,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972249945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:29,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:29,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:29,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:29,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:29,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:29,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:29,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:29,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972249947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:29,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:29,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:29,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:29,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:29,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1730972249949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:29,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:29,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:29,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:29,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:29,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972249951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:29,958 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:29,958 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:29,958 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:30,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:30,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972250053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:30,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:30,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972250053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:30,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:30,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:30,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:30,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:30,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:30,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:30,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:30,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972250063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:30,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:30,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:30,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:30,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:30,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1730972250063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:30,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:30,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:30,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:30,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:30,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972250063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:30,065 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:30,065 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:30,065 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:30,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:30,217 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107da66a09f3b2349de9d1dee53bb530780_81b216f4bb32d787c444da8261b91891 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107da66a09f3b2349de9d1dee53bb530780_81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:30,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/1e5cf4a271af48f1b590d6b4b2f874d6, store: [table=TestAcidGuarantees family=A region=81b216f4bb32d787c444da8261b91891] 2024-11-07T09:36:30,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/1e5cf4a271af48f1b590d6b4b2f874d6 is 175, key is test_row_0/A:col10/1730972189585/Put/seqid=0 2024-11-07T09:36:30,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741972_1148 (size=30955) 2024-11-07T09:36:30,234 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/1e5cf4a271af48f1b590d6b4b2f874d6 2024-11-07T09:36:30,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/23fe023d822342f7a7c73cc0aaa140a1 is 50, key is test_row_0/B:col10/1730972189585/Put/seqid=0 2024-11-07T09:36:30,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741973_1149 (size=12001) 2024-11-07T09:36:30,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:30,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972250263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:30,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:30,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972250263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:30,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:30,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:30,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:30,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:30,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:30,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:30,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-07T09:36:30,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:30,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972250273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:30,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:30,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:30,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:30,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1730972250273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:30,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:30,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:30,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:30,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:30,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:30,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972250273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:30,281 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:30,281 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:30,281 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:30,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:30,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972250573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:30,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:30,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:30,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:30,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:30,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972250573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:30,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:30,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:30,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:30,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:30,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1730972250583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:30,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:30,589 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:30,589 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:30,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972250583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:30,589 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:30,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:30,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:30,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:30,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:30,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972250593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:30,594 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:30,595 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:30,595 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:30,659 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/23fe023d822342f7a7c73cc0aaa140a1 2024-11-07T09:36:30,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/a4f7741fa20b48f09130f9a4cf8f7c25 is 50, key is test_row_0/C:col10/1730972189585/Put/seqid=0 2024-11-07T09:36:30,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741974_1150 (size=12001) 2024-11-07T09:36:30,837 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T09:36:31,076 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/a4f7741fa20b48f09130f9a4cf8f7c25 2024-11-07T09:36:31,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/1e5cf4a271af48f1b590d6b4b2f874d6 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/1e5cf4a271af48f1b590d6b4b2f874d6 2024-11-07T09:36:31,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:31,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:31,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972251084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:31,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972251084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:31,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:31,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:31,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:31,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:31,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:31,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:31,093 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/1e5cf4a271af48f1b590d6b4b2f874d6, entries=150, sequenceid=54, filesize=30.2 K 2024-11-07T09:36:31,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/23fe023d822342f7a7c73cc0aaa140a1 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/23fe023d822342f7a7c73cc0aaa140a1 2024-11-07T09:36:31,099 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/23fe023d822342f7a7c73cc0aaa140a1, entries=150, sequenceid=54, filesize=11.7 K 2024-11-07T09:36:31,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/a4f7741fa20b48f09130f9a4cf8f7c25 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/a4f7741fa20b48f09130f9a4cf8f7c25 2024-11-07T09:36:31,106 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/a4f7741fa20b48f09130f9a4cf8f7c25, entries=150, sequenceid=54, filesize=11.7 K 2024-11-07T09:36:31,107 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 81b216f4bb32d787c444da8261b91891 in 1348ms, sequenceid=54, compaction requested=true 2024-11-07T09:36:31,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 81b216f4bb32d787c444da8261b91891: 2024-11-07T09:36:31,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:31,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-07T09:36:31,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-07T09:36:31,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:31,108 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:36:31,108 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81b216f4bb32d787c444da8261b91891 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-07T09:36:31,114 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-07T09:36:31,114 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9710 sec 2024-11-07T09:36:31,116 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.9800 sec 2024-11-07T09:36:31,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411071b1a7d6af3d54283a7ff42cc9642dc7d_81b216f4bb32d787c444da8261b91891 is 50, key is test_row_0/A:col10/1730972189945/Put/seqid=0 2024-11-07T09:36:31,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:31,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972251127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:31,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:31,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:31,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:31,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:31,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972251130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:31,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:31,139 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:31,139 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:31,139 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:31,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1730972251130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:31,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:31,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:31,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:31,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741975_1151 (size=12154) 2024-11-07T09:36:31,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:31,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972251243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:31,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:31,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:31,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:31,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972251243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:31,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:31,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:31,250 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:31,250 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:31,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1730972251243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:31,250 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:31,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:31,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:31,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:31,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-07T09:36:31,284 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:36:31,284 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T09:36:31,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-07T09:36:31,287 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T09:36:31,287 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T09:36:31,288 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T09:36:31,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-07T09:36:31,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-07T09:36:31,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-07T09:36:31,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:31,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. as already flushing 2024-11-07T09:36:31,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:31,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:31,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:31,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:31,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:31,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972251453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:31,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:31,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972251454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:31,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:31,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:31,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:31,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:31,459 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:31,459 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:31,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1730972251454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:31,459 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:31,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:31,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:31,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:31,556 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:31,562 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411071b1a7d6af3d54283a7ff42cc9642dc7d_81b216f4bb32d787c444da8261b91891 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411071b1a7d6af3d54283a7ff42cc9642dc7d_81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:31,563 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/3c8ef269723744dda71418c66bb09a37, store: [table=TestAcidGuarantees family=A region=81b216f4bb32d787c444da8261b91891] 2024-11-07T09:36:31,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/3c8ef269723744dda71418c66bb09a37 is 175, key is test_row_0/A:col10/1730972189945/Put/seqid=0 2024-11-07T09:36:31,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741976_1152 (size=30955) 2024-11-07T09:36:31,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-07T09:36:31,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:31,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. as already flushing 2024-11-07T09:36:31,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:31,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:31,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:31,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:31,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-07T09:36:31,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-07T09:36:31,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:31,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. as already flushing 2024-11-07T09:36:31,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:31,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:31,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:31,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:31,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:31,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972251763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:31,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:31,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:31,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:31,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:31,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972251763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:31,766 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:31,766 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:31,766 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:31,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:31,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1730972251763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:31,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:31,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:31,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:31,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-07T09:36:31,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:31,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. as already flushing 2024-11-07T09:36:31,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:31,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:31,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:31,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:31,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-07T09:36:31,974 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=80, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/3c8ef269723744dda71418c66bb09a37 2024-11-07T09:36:31,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/b2ef66f1ceec4c778433d762171d1c3b is 50, key is test_row_0/B:col10/1730972189945/Put/seqid=0 2024-11-07T09:36:32,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741977_1153 (size=12001) 2024-11-07T09:36:32,012 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/b2ef66f1ceec4c778433d762171d1c3b 2024-11-07T09:36:32,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/e0d3ad31220c4902a7f129210a576710 is 50, key is test_row_0/C:col10/1730972189945/Put/seqid=0 2024-11-07T09:36:32,058 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-07T09:36:32,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:32,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. as already flushing 2024-11-07T09:36:32,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741978_1154 (size=12001) 2024-11-07T09:36:32,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:32,059 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:32,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:32,059 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/e0d3ad31220c4902a7f129210a576710 2024-11-07T09:36:32,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:32,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/3c8ef269723744dda71418c66bb09a37 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/3c8ef269723744dda71418c66bb09a37 2024-11-07T09:36:32,074 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/3c8ef269723744dda71418c66bb09a37, entries=150, sequenceid=80, filesize=30.2 K 2024-11-07T09:36:32,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/b2ef66f1ceec4c778433d762171d1c3b as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/b2ef66f1ceec4c778433d762171d1c3b 2024-11-07T09:36:32,086 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/b2ef66f1ceec4c778433d762171d1c3b, entries=150, sequenceid=80, filesize=11.7 K 2024-11-07T09:36:32,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/e0d3ad31220c4902a7f129210a576710 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/e0d3ad31220c4902a7f129210a576710 2024-11-07T09:36:32,095 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/e0d3ad31220c4902a7f129210a576710, entries=150, sequenceid=80, filesize=11.7 K 2024-11-07T09:36:32,096 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 81b216f4bb32d787c444da8261b91891 in 988ms, sequenceid=80, compaction requested=true 2024-11-07T09:36:32,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81b216f4bb32d787c444da8261b91891: 2024-11-07T09:36:32,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81b216f4bb32d787c444da8261b91891:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T09:36:32,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:32,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81b216f4bb32d787c444da8261b91891:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T09:36:32,097 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T09:36:32,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:36:32,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81b216f4bb32d787c444da8261b91891:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:36:32,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T09:36:32,097 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T09:36:32,099 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 149594 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T09:36:32,099 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 81b216f4bb32d787c444da8261b91891/A is initiating minor compaction (all files) 2024-11-07T09:36:32,099 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81b216f4bb32d787c444da8261b91891/A in TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:32,099 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/f41795ffda244dceb7871ee1d6f14c41, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/287bfdbe7ce7446c97d55f3bcc8cdbf2, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/1e5cf4a271af48f1b590d6b4b2f874d6, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/3c8ef269723744dda71418c66bb09a37] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp, totalSize=146.1 K 2024-11-07T09:36:32,099 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:32,099 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. files: [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/f41795ffda244dceb7871ee1d6f14c41, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/287bfdbe7ce7446c97d55f3bcc8cdbf2, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/1e5cf4a271af48f1b590d6b4b2f874d6, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/3c8ef269723744dda71418c66bb09a37] 2024-11-07T09:36:32,100 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T09:36:32,100 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 81b216f4bb32d787c444da8261b91891/B is initiating minor compaction (all files) 2024-11-07T09:36:32,100 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81b216f4bb32d787c444da8261b91891/B in TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:32,100 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/5d5aea263ee84112bb451e1fcbb5ea18, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/0ca5866ee0d34230b06b405614e2ef45, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/23fe023d822342f7a7c73cc0aaa140a1, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/b2ef66f1ceec4c778433d762171d1c3b] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp, totalSize=44.6 K 2024-11-07T09:36:32,101 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d5aea263ee84112bb451e1fcbb5ea18, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1730972189165 2024-11-07T09:36:32,101 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting f41795ffda244dceb7871ee1d6f14c41, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1730972189152 2024-11-07T09:36:32,102 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:36:32,102 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81b216f4bb32d787c444da8261b91891 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T09:36:32,102 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 287bfdbe7ce7446c97d55f3bcc8cdbf2, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1730972189230 2024-11-07T09:36:32,102 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ca5866ee0d34230b06b405614e2ef45, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1730972189235 2024-11-07T09:36:32,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:32,105 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 23fe023d822342f7a7c73cc0aaa140a1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1730972189574 2024-11-07T09:36:32,105 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting b2ef66f1ceec4c778433d762171d1c3b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1730972189945 2024-11-07T09:36:32,106 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e5cf4a271af48f1b590d6b4b2f874d6, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1730972189574 2024-11-07T09:36:32,107 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c8ef269723744dda71418c66bb09a37, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1730972189945 2024-11-07T09:36:32,130 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=81b216f4bb32d787c444da8261b91891] 2024-11-07T09:36:32,131 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81b216f4bb32d787c444da8261b91891#B#compaction#135 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:32,131 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110721d85f62bf45441c9c730cbf6dc28c62_81b216f4bb32d787c444da8261b91891 is 50, key is test_row_0/A:col10/1730972192100/Put/seqid=0 2024-11-07T09:36:32,132 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/82eb2e0b0c074c49a4dfc03998308af2 is 50, key is test_row_0/B:col10/1730972189945/Put/seqid=0 2024-11-07T09:36:32,154 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107e38018e047634e16afee2982654d667a_81b216f4bb32d787c444da8261b91891 store=[table=TestAcidGuarantees family=A region=81b216f4bb32d787c444da8261b91891] 2024-11-07T09:36:32,158 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107e38018e047634e16afee2982654d667a_81b216f4bb32d787c444da8261b91891, store=[table=TestAcidGuarantees family=A region=81b216f4bb32d787c444da8261b91891] 2024-11-07T09:36:32,158 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107e38018e047634e16afee2982654d667a_81b216f4bb32d787c444da8261b91891 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=81b216f4bb32d787c444da8261b91891] 2024-11-07T09:36:32,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741979_1155 (size=14594) 2024-11-07T09:36:32,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741980_1156 (size=12139) 2024-11-07T09:36:32,172 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:32,178 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110721d85f62bf45441c9c730cbf6dc28c62_81b216f4bb32d787c444da8261b91891 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110721d85f62bf45441c9c730cbf6dc28c62_81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:32,179 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/24b53a4627684be99e8c00822e6f214c, store: [table=TestAcidGuarantees family=A region=81b216f4bb32d787c444da8261b91891] 2024-11-07T09:36:32,180 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/24b53a4627684be99e8c00822e6f214c is 175, key is test_row_0/A:col10/1730972192100/Put/seqid=0 2024-11-07T09:36:32,181 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/82eb2e0b0c074c49a4dfc03998308af2 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/82eb2e0b0c074c49a4dfc03998308af2 2024-11-07T09:36:32,187 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 81b216f4bb32d787c444da8261b91891/B of 81b216f4bb32d787c444da8261b91891 into 82eb2e0b0c074c49a4dfc03998308af2(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:32,187 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81b216f4bb32d787c444da8261b91891: 2024-11-07T09:36:32,187 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., storeName=81b216f4bb32d787c444da8261b91891/B, priority=12, startTime=1730972192097; duration=0sec 2024-11-07T09:36:32,188 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:36:32,188 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81b216f4bb32d787c444da8261b91891:B 2024-11-07T09:36:32,188 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T09:36:32,189 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T09:36:32,189 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 81b216f4bb32d787c444da8261b91891/C is initiating minor compaction (all files) 2024-11-07T09:36:32,189 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81b216f4bb32d787c444da8261b91891/C in TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:32,189 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/956d40ed423c4871aad9f07422e6709e, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/c2bc84f0e2b34f20848921d0a69d3487, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/a4f7741fa20b48f09130f9a4cf8f7c25, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/e0d3ad31220c4902a7f129210a576710] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp, totalSize=44.6 K 2024-11-07T09:36:32,190 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 956d40ed423c4871aad9f07422e6709e, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1730972189165 2024-11-07T09:36:32,191 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting c2bc84f0e2b34f20848921d0a69d3487, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1730972189235 2024-11-07T09:36:32,191 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting a4f7741fa20b48f09130f9a4cf8f7c25, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1730972189574 2024-11-07T09:36:32,192 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting e0d3ad31220c4902a7f129210a576710, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1730972189945 2024-11-07T09:36:32,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741982_1158 (size=39549) 2024-11-07T09:36:32,210 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=91, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/24b53a4627684be99e8c00822e6f214c 2024-11-07T09:36:32,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-07T09:36:32,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:32,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. as already flushing 2024-11-07T09:36:32,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:32,213 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:32,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:32,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:32,227 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81b216f4bb32d787c444da8261b91891#C#compaction#138 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:32,228 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/71a2cea874a0484a81e0d8eae5dca57e is 50, key is test_row_0/C:col10/1730972189945/Put/seqid=0 2024-11-07T09:36:32,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741981_1157 (size=4469) 2024-11-07T09:36:32,238 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81b216f4bb32d787c444da8261b91891#A#compaction#137 average throughput is 0.23 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:32,239 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/0c367ad1dea44e78956743da27f3f767 is 175, key is test_row_0/A:col10/1730972189945/Put/seqid=0 2024-11-07T09:36:32,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/d0efd62a5a204250bd84425a73c9d9f6 is 50, key is test_row_0/B:col10/1730972192100/Put/seqid=0 2024-11-07T09:36:32,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:32,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972252280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:32,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:32,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:32,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:32,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972252281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:32,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:32,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:32,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:32,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:32,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741985_1161 (size=12001) 2024-11-07T09:36:32,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/d0efd62a5a204250bd84425a73c9d9f6 2024-11-07T09:36:32,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:32,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972252283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:32,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741983_1159 (size=12139) 2024-11-07T09:36:32,298 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:32,298 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:32,298 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:32,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:32,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1730972252284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:32,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:32,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972252286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:32,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:32,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:32,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:32,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:32,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:32,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:32,307 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/71a2cea874a0484a81e0d8eae5dca57e as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/71a2cea874a0484a81e0d8eae5dca57e 2024-11-07T09:36:32,314 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 81b216f4bb32d787c444da8261b91891/C of 81b216f4bb32d787c444da8261b91891 into 71a2cea874a0484a81e0d8eae5dca57e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:32,314 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81b216f4bb32d787c444da8261b91891: 2024-11-07T09:36:32,314 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., storeName=81b216f4bb32d787c444da8261b91891/C, priority=12, startTime=1730972192097; duration=0sec 2024-11-07T09:36:32,314 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:32,314 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81b216f4bb32d787c444da8261b91891:C 2024-11-07T09:36:32,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741984_1160 (size=31093) 2024-11-07T09:36:32,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/bd73294e814841ec8a5d32e285d44e84 is 50, key is test_row_0/C:col10/1730972192100/Put/seqid=0 2024-11-07T09:36:32,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741986_1162 (size=12001) 2024-11-07T09:36:32,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-07T09:36:32,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:32,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. as already flushing 2024-11-07T09:36:32,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:32,367 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:32,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:32,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:32,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:32,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972252393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:32,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:32,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972252393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:32,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:32,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:32,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:32,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:32,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:32,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:32,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:32,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972252403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:32,404 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:32,404 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:32,404 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:32,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:32,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972252404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:32,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:32,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:32,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:32,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:32,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1730972252413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:32,417 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:32,417 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:32,417 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:32,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-07T09:36:32,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-07T09:36:32,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:32,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. as already flushing 2024-11-07T09:36:32,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:32,521 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:32,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:32,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:32,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:32,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972252603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:32,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:32,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:32,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972252603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:32,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:32,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:32,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:32,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:32,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:32,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:32,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:32,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972252613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:32,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972252613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:32,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:32,618 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:32,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:32,618 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:32,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:32,618 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:32,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:32,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1730972252623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:32,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:32,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:32,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:32,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-07T09:36:32,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:32,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. as already flushing 2024-11-07T09:36:32,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:32,675 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:32,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:32,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:32,730 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/0c367ad1dea44e78956743da27f3f767 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/0c367ad1dea44e78956743da27f3f767 2024-11-07T09:36:32,736 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 81b216f4bb32d787c444da8261b91891/A of 81b216f4bb32d787c444da8261b91891 into 0c367ad1dea44e78956743da27f3f767(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:32,736 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81b216f4bb32d787c444da8261b91891: 2024-11-07T09:36:32,737 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., storeName=81b216f4bb32d787c444da8261b91891/A, priority=12, startTime=1730972192097; duration=0sec 2024-11-07T09:36:32,737 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:32,737 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81b216f4bb32d787c444da8261b91891:A 2024-11-07T09:36:32,768 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/bd73294e814841ec8a5d32e285d44e84 2024-11-07T09:36:32,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/24b53a4627684be99e8c00822e6f214c as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/24b53a4627684be99e8c00822e6f214c 2024-11-07T09:36:32,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/24b53a4627684be99e8c00822e6f214c, entries=200, sequenceid=91, filesize=38.6 K 2024-11-07T09:36:32,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/d0efd62a5a204250bd84425a73c9d9f6 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/d0efd62a5a204250bd84425a73c9d9f6 2024-11-07T09:36:32,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/d0efd62a5a204250bd84425a73c9d9f6, entries=150, sequenceid=91, filesize=11.7 K 2024-11-07T09:36:32,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/bd73294e814841ec8a5d32e285d44e84 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/bd73294e814841ec8a5d32e285d44e84 2024-11-07T09:36:32,793 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/bd73294e814841ec8a5d32e285d44e84, entries=150, sequenceid=91, filesize=11.7 K 2024-11-07T09:36:32,795 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 81b216f4bb32d787c444da8261b91891 in 693ms, sequenceid=91, compaction requested=false 2024-11-07T09:36:32,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81b216f4bb32d787c444da8261b91891: 2024-11-07T09:36:32,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-07T09:36:32,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:32,829 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 81b216f4bb32d787c444da8261b91891 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-07T09:36:32,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107c438d53b4b6648a1a4e2f7dce5bbc856_81b216f4bb32d787c444da8261b91891 is 50, key is test_row_0/A:col10/1730972192277/Put/seqid=0 2024-11-07T09:36:32,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741987_1163 (size=12154) 2024-11-07T09:36:32,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:32,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. as already flushing 2024-11-07T09:36:32,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:32,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972252923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:32,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:32,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:32,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:32,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:32,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972252929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:32,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:32,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:32,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:32,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972252930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:32,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:32,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:32,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972252930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:32,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:32,936 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:32,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1730972252933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:32,936 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:32,936 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:32,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:32,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:32,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:32,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:32,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:32,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:33,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:33,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972253033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:33,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:33,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:33,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:33,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:33,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972253043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:33,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:33,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972253043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:33,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:33,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972253043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:33,047 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:33,047 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:33,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:33,047 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:33,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:33,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:33,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:33,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:33,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:33,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:33,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972253243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:33,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:33,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:33,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:33,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:33,255 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107c438d53b4b6648a1a4e2f7dce5bbc856_81b216f4bb32d787c444da8261b91891 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107c438d53b4b6648a1a4e2f7dce5bbc856_81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:33,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:33,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972253253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:33,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:33,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972253253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:33,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:33,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:33,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972253254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:33,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:33,256 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:33,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:33,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:33,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:33,256 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:33,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:33,256 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:33,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/a19c6b5148644e78919a17b21d9bfad9, store: [table=TestAcidGuarantees family=A region=81b216f4bb32d787c444da8261b91891] 2024-11-07T09:36:33,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/a19c6b5148644e78919a17b21d9bfad9 is 175, key is test_row_0/A:col10/1730972192277/Put/seqid=0 2024-11-07T09:36:33,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741988_1164 (size=30955) 2024-11-07T09:36:33,269 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=120, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/a19c6b5148644e78919a17b21d9bfad9 2024-11-07T09:36:33,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/8a938028424d4b3d8def87f501151cf4 is 50, key is test_row_0/B:col10/1730972192277/Put/seqid=0 2024-11-07T09:36:33,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741989_1165 (size=12001) 2024-11-07T09:36:33,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-07T09:36:33,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:33,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1730972253444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:33,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:33,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:33,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:33,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:33,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972253553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:33,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:33,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:33,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:33,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:33,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972253563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:33,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:33,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972253563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:33,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:33,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:33,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:33,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:33,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972253563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:33,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:33,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:33,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:33,570 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:33,570 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:33,570 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:33,695 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/8a938028424d4b3d8def87f501151cf4 2024-11-07T09:36:33,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/c2849a1bc35544a99db9d96bff0d909d is 50, key is test_row_0/C:col10/1730972192277/Put/seqid=0 2024-11-07T09:36:33,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741990_1166 (size=12001) 2024-11-07T09:36:33,715 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/c2849a1bc35544a99db9d96bff0d909d 2024-11-07T09:36:33,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/a19c6b5148644e78919a17b21d9bfad9 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/a19c6b5148644e78919a17b21d9bfad9 2024-11-07T09:36:33,733 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/a19c6b5148644e78919a17b21d9bfad9, entries=150, sequenceid=120, filesize=30.2 K 2024-11-07T09:36:33,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/8a938028424d4b3d8def87f501151cf4 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/8a938028424d4b3d8def87f501151cf4 2024-11-07T09:36:33,740 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/8a938028424d4b3d8def87f501151cf4, entries=150, sequenceid=120, filesize=11.7 K 2024-11-07T09:36:33,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/c2849a1bc35544a99db9d96bff0d909d as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/c2849a1bc35544a99db9d96bff0d909d 2024-11-07T09:36:33,749 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/c2849a1bc35544a99db9d96bff0d909d, entries=150, sequenceid=120, filesize=11.7 K 2024-11-07T09:36:33,751 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 81b216f4bb32d787c444da8261b91891 in 923ms, sequenceid=120, compaction requested=true 2024-11-07T09:36:33,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 81b216f4bb32d787c444da8261b91891: 2024-11-07T09:36:33,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:33,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-07T09:36:33,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-07T09:36:33,757 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-07T09:36:33,757 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4650 sec 2024-11-07T09:36:33,761 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.4740 sec 2024-11-07T09:36:34,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:34,064 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:36:34,065 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81b216f4bb32d787c444da8261b91891 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T09:36:34,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107ca23ad9485d74f9bad5981ff256f462c_81b216f4bb32d787c444da8261b91891 is 50, key is test_row_0/A:col10/1730972192927/Put/seqid=0 2024-11-07T09:36:34,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741991_1167 (size=12254) 2024-11-07T09:36:34,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:34,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972254108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:34,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:34,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:34,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:34,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:34,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972254109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:34,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:34,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:34,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:34,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:34,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972254110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:34,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:34,119 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:34,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972254111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:34,119 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:34,119 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:34,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:34,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:34,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:34,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:34,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972254223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:34,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:34,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972254223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:34,227 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:34,227 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:34,227 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:34,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:34,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972254223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:34,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:34,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:34,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:34,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:34,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972254223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:34,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:34,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:34,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:34,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:34,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:34,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:34,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:34,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972254433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:34,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:34,436 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:34,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972254433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:34,436 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:34,436 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:34,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:34,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972254433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:34,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:34,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:34,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:34,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:34,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:34,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:34,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:34,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972254443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:34,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:34,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:34,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:34,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:34,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1730972254453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:34,457 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 9f1eb259a29b,42895,1730972153970 for 'test_row_0' in 81b216f4bb32d787c444da8261b91891 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T09:36:34,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:34,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:34,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:34,481 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:34,488 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107ca23ad9485d74f9bad5981ff256f462c_81b216f4bb32d787c444da8261b91891 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107ca23ad9485d74f9bad5981ff256f462c_81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:34,490 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/4fd68b55d7cd4678a5aa25735a6c8ef4, store: [table=TestAcidGuarantees family=A region=81b216f4bb32d787c444da8261b91891] 2024-11-07T09:36:34,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/4fd68b55d7cd4678a5aa25735a6c8ef4 is 175, key is test_row_0/A:col10/1730972192927/Put/seqid=0 2024-11-07T09:36:34,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741992_1168 (size=31055) 2024-11-07T09:36:34,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:34,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972254743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:34,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:34,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:34,749 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:34,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972254743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:34,749 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:34,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972254743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:34,749 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:34,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:34,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:34,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:34,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:34,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:34,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:34,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:34,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972254763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:34,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:34,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:34,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:34,901 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=131, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/4fd68b55d7cd4678a5aa25735a6c8ef4 2024-11-07T09:36:34,913 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/99b30512f91c448996adb7f004a9f8b4 is 50, key is test_row_0/B:col10/1730972192927/Put/seqid=0 2024-11-07T09:36:34,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741993_1169 (size=12101) 2024-11-07T09:36:34,923 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/99b30512f91c448996adb7f004a9f8b4 2024-11-07T09:36:34,933 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/25ab72fa59724b6dae8eb1082fbcf476 is 50, key is test_row_0/C:col10/1730972192927/Put/seqid=0 2024-11-07T09:36:34,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741994_1170 (size=12101) 2024-11-07T09:36:35,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:35,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972255263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:35,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:35,269 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:35,269 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:35,269 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:35,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972255263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:35,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:35,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:35,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:35,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:35,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972255263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:35,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:35,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:35,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:35,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:35,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972255273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:35,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:35,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:35,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:35,343 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/25ab72fa59724b6dae8eb1082fbcf476 2024-11-07T09:36:35,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/4fd68b55d7cd4678a5aa25735a6c8ef4 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/4fd68b55d7cd4678a5aa25735a6c8ef4 2024-11-07T09:36:35,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/4fd68b55d7cd4678a5aa25735a6c8ef4, entries=150, sequenceid=131, filesize=30.3 K 2024-11-07T09:36:35,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/99b30512f91c448996adb7f004a9f8b4 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/99b30512f91c448996adb7f004a9f8b4 2024-11-07T09:36:35,359 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/99b30512f91c448996adb7f004a9f8b4, entries=150, sequenceid=131, filesize=11.8 K 2024-11-07T09:36:35,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/25ab72fa59724b6dae8eb1082fbcf476 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/25ab72fa59724b6dae8eb1082fbcf476 2024-11-07T09:36:35,366 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/25ab72fa59724b6dae8eb1082fbcf476, entries=150, sequenceid=131, filesize=11.8 K 2024-11-07T09:36:35,367 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 81b216f4bb32d787c444da8261b91891 in 1302ms, sequenceid=131, compaction requested=true 2024-11-07T09:36:35,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81b216f4bb32d787c444da8261b91891: 2024-11-07T09:36:35,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81b216f4bb32d787c444da8261b91891:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T09:36:35,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:35,367 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T09:36:35,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81b216f4bb32d787c444da8261b91891:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T09:36:35,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:35,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81b216f4bb32d787c444da8261b91891:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:36:35,368 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T09:36:35,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:36:35,372 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132652 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T09:36:35,372 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48242 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T09:36:35,372 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 81b216f4bb32d787c444da8261b91891/A is initiating minor compaction (all files) 2024-11-07T09:36:35,372 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 81b216f4bb32d787c444da8261b91891/B is initiating minor compaction (all files) 2024-11-07T09:36:35,372 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81b216f4bb32d787c444da8261b91891/B in TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:35,372 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81b216f4bb32d787c444da8261b91891/A in TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:35,372 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/82eb2e0b0c074c49a4dfc03998308af2, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/d0efd62a5a204250bd84425a73c9d9f6, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/8a938028424d4b3d8def87f501151cf4, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/99b30512f91c448996adb7f004a9f8b4] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp, totalSize=47.1 K 2024-11-07T09:36:35,372 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/0c367ad1dea44e78956743da27f3f767, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/24b53a4627684be99e8c00822e6f214c, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/a19c6b5148644e78919a17b21d9bfad9, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/4fd68b55d7cd4678a5aa25735a6c8ef4] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp, totalSize=129.5 K 2024-11-07T09:36:35,372 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:35,372 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. files: [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/0c367ad1dea44e78956743da27f3f767, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/24b53a4627684be99e8c00822e6f214c, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/a19c6b5148644e78919a17b21d9bfad9, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/4fd68b55d7cd4678a5aa25735a6c8ef4] 2024-11-07T09:36:35,373 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 82eb2e0b0c074c49a4dfc03998308af2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1730972189945 2024-11-07T09:36:35,373 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c367ad1dea44e78956743da27f3f767, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1730972189945 2024-11-07T09:36:35,374 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24b53a4627684be99e8c00822e6f214c, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1730972191120 2024-11-07T09:36:35,374 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting d0efd62a5a204250bd84425a73c9d9f6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1730972191120 2024-11-07T09:36:35,374 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting a19c6b5148644e78919a17b21d9bfad9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1730972192277 2024-11-07T09:36:35,374 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a938028424d4b3d8def87f501151cf4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1730972192277 2024-11-07T09:36:35,375 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 99b30512f91c448996adb7f004a9f8b4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1730972192927 2024-11-07T09:36:35,375 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fd68b55d7cd4678a5aa25735a6c8ef4, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1730972192927 2024-11-07T09:36:35,385 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=81b216f4bb32d787c444da8261b91891] 2024-11-07T09:36:35,388 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81b216f4bb32d787c444da8261b91891#B#compaction#148 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:35,389 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/19b7869e32f84344a9c082a316a8479c is 50, key is test_row_0/B:col10/1730972192927/Put/seqid=0 2024-11-07T09:36:35,390 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107f41fd4a147c744eaa0b9e788d401f76d_81b216f4bb32d787c444da8261b91891 store=[table=TestAcidGuarantees family=A region=81b216f4bb32d787c444da8261b91891] 2024-11-07T09:36:35,395 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107f41fd4a147c744eaa0b9e788d401f76d_81b216f4bb32d787c444da8261b91891, store=[table=TestAcidGuarantees family=A region=81b216f4bb32d787c444da8261b91891] 2024-11-07T09:36:35,395 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107f41fd4a147c744eaa0b9e788d401f76d_81b216f4bb32d787c444da8261b91891 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=81b216f4bb32d787c444da8261b91891] 2024-11-07T09:36:35,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741996_1172 (size=4469) 2024-11-07T09:36:35,424 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81b216f4bb32d787c444da8261b91891#A#compaction#147 average throughput is 0.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:35,425 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/8fc5b5bd41ff4409abb99c8777732f72 is 175, key is test_row_0/A:col10/1730972192927/Put/seqid=0 2024-11-07T09:36:35,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741995_1171 (size=12375) 2024-11-07T09:36:35,455 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/19b7869e32f84344a9c082a316a8479c as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/19b7869e32f84344a9c082a316a8479c 2024-11-07T09:36:35,463 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 81b216f4bb32d787c444da8261b91891/B of 81b216f4bb32d787c444da8261b91891 into 19b7869e32f84344a9c082a316a8479c(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:35,464 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81b216f4bb32d787c444da8261b91891: 2024-11-07T09:36:35,464 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., storeName=81b216f4bb32d787c444da8261b91891/B, priority=12, startTime=1730972195367; duration=0sec 2024-11-07T09:36:35,464 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:36:35,464 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81b216f4bb32d787c444da8261b91891:B 2024-11-07T09:36:35,464 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T09:36:35,467 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48242 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T09:36:35,467 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 81b216f4bb32d787c444da8261b91891/C is initiating minor compaction (all files) 2024-11-07T09:36:35,467 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81b216f4bb32d787c444da8261b91891/C in TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:35,468 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/71a2cea874a0484a81e0d8eae5dca57e, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/bd73294e814841ec8a5d32e285d44e84, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/c2849a1bc35544a99db9d96bff0d909d, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/25ab72fa59724b6dae8eb1082fbcf476] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp, totalSize=47.1 K 2024-11-07T09:36:35,468 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 71a2cea874a0484a81e0d8eae5dca57e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1730972189945 2024-11-07T09:36:35,469 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting bd73294e814841ec8a5d32e285d44e84, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1730972191120 2024-11-07T09:36:35,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741997_1173 (size=31329) 2024-11-07T09:36:35,472 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting c2849a1bc35544a99db9d96bff0d909d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1730972192277 2024-11-07T09:36:35,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-07T09:36:35,473 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 25ab72fa59724b6dae8eb1082fbcf476, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1730972192927 2024-11-07T09:36:35,474 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:36:35,474 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T09:36:35,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-07T09:36:35,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-07T09:36:35,477 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T09:36:35,478 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T09:36:35,478 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T09:36:35,481 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/8fc5b5bd41ff4409abb99c8777732f72 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/8fc5b5bd41ff4409abb99c8777732f72 2024-11-07T09:36:35,488 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 81b216f4bb32d787c444da8261b91891/A of 81b216f4bb32d787c444da8261b91891 into 8fc5b5bd41ff4409abb99c8777732f72(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:35,488 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81b216f4bb32d787c444da8261b91891: 2024-11-07T09:36:35,488 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., storeName=81b216f4bb32d787c444da8261b91891/A, priority=12, startTime=1730972195367; duration=0sec 2024-11-07T09:36:35,488 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:35,488 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81b216f4bb32d787c444da8261b91891:A 2024-11-07T09:36:35,503 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81b216f4bb32d787c444da8261b91891#C#compaction#149 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:35,504 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/6df39d9e04834ea99ca6de2fc2c9f7b9 is 50, key is test_row_0/C:col10/1730972192927/Put/seqid=0 2024-11-07T09:36:35,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741998_1174 (size=12375) 2024-11-07T09:36:35,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-07T09:36:35,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-07T09:36:35,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:35,631 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 81b216f4bb32d787c444da8261b91891 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T09:36:35,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110714ef3a86f1a24b4bb0359a26d0c33432_81b216f4bb32d787c444da8261b91891 is 50, key is test_row_0/A:col10/1730972194110/Put/seqid=0 2024-11-07T09:36:35,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741999_1175 (size=12304) 2024-11-07T09:36:35,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-07T09:36:35,930 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/6df39d9e04834ea99ca6de2fc2c9f7b9 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/6df39d9e04834ea99ca6de2fc2c9f7b9 2024-11-07T09:36:35,936 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 81b216f4bb32d787c444da8261b91891/C of 81b216f4bb32d787c444da8261b91891 into 6df39d9e04834ea99ca6de2fc2c9f7b9(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:35,936 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81b216f4bb32d787c444da8261b91891: 2024-11-07T09:36:35,937 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., storeName=81b216f4bb32d787c444da8261b91891/C, priority=12, startTime=1730972195367; duration=0sec 2024-11-07T09:36:35,937 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:35,937 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81b216f4bb32d787c444da8261b91891:C 2024-11-07T09:36:36,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:36,089 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110714ef3a86f1a24b4bb0359a26d0c33432_81b216f4bb32d787c444da8261b91891 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110714ef3a86f1a24b4bb0359a26d0c33432_81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:36,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/cdf7c281a874458598e364ea88f18867, store: [table=TestAcidGuarantees family=A region=81b216f4bb32d787c444da8261b91891] 2024-11-07T09:36:36,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/cdf7c281a874458598e364ea88f18867 is 175, key is test_row_0/A:col10/1730972194110/Put/seqid=0 2024-11-07T09:36:36,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742000_1176 (size=31105) 2024-11-07T09:36:36,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-07T09:36:36,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:36,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. as already flushing 2024-11-07T09:36:36,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:36,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972256290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:36,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:36,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:36,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:36,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:36,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972256296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:36,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:36,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972256298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:36,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:36,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:36,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:36,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972256299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:36,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:36,304 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:36,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:36,304 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:36,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:36,304 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:36,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:36,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:36,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972256403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:36,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:36,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:36,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:36,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:36,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972256413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:36,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:36,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:36,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:36,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972256413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:36,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:36,415 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:36,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:36,415 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:36,415 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:36,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972256413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:36,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:36,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:36,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:36,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:36,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1730972256482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:36,486 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 9f1eb259a29b,42895,1730972153970 for 'test_row_0' in 81b216f4bb32d787c444da8261b91891 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T09:36:36,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:36,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:36,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:36,496 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=158, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/cdf7c281a874458598e364ea88f18867 2024-11-07T09:36:36,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/1cbde7ee907d4aff9751fe9f838874f3 is 50, key is test_row_0/B:col10/1730972194110/Put/seqid=0 2024-11-07T09:36:36,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742001_1177 (size=12151) 2024-11-07T09:36:36,518 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/1cbde7ee907d4aff9751fe9f838874f3 2024-11-07T09:36:36,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/ed717274184c4321ad515ea731136aad is 50, key is test_row_0/C:col10/1730972194110/Put/seqid=0 2024-11-07T09:36:36,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742002_1178 (size=12151) 2024-11-07T09:36:36,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-07T09:36:36,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:36,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972256613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:36,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:36,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:36,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:36,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:36,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972256623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:36,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:36,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:36,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:36,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:36,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972256623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:36,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:36,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972256623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:36,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:36,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:36,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:36,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:36,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:36,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:36,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:36,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972256922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:36,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:36,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:36,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:36,933 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/ed717274184c4321ad515ea731136aad 2024-11-07T09:36:36,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:36,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972256933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:36,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:36,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:36,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:36,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972256933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:36,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:36,938 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:36,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:36,938 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:36,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972256933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:36,938 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:36,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:36,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:36,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:36,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/cdf7c281a874458598e364ea88f18867 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/cdf7c281a874458598e364ea88f18867 2024-11-07T09:36:36,944 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/cdf7c281a874458598e364ea88f18867, entries=150, sequenceid=158, filesize=30.4 K 2024-11-07T09:36:36,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/1cbde7ee907d4aff9751fe9f838874f3 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/1cbde7ee907d4aff9751fe9f838874f3 2024-11-07T09:36:36,952 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/1cbde7ee907d4aff9751fe9f838874f3, entries=150, sequenceid=158, filesize=11.9 K 2024-11-07T09:36:36,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/ed717274184c4321ad515ea731136aad as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/ed717274184c4321ad515ea731136aad 2024-11-07T09:36:36,960 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/ed717274184c4321ad515ea731136aad, entries=150, sequenceid=158, filesize=11.9 K 2024-11-07T09:36:36,961 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 81b216f4bb32d787c444da8261b91891 in 1330ms, sequenceid=158, compaction requested=false 2024-11-07T09:36:36,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 81b216f4bb32d787c444da8261b91891: 2024-11-07T09:36:36,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:36,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-07T09:36:36,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-07T09:36:36,965 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-07T09:36:36,965 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4840 sec 2024-11-07T09:36:36,968 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.4920 sec 2024-11-07T09:36:37,435 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:36:37,435 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81b216f4bb32d787c444da8261b91891 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-07T09:36:37,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:37,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411079df2a4dd6c6e4fab891ccc186ffac383_81b216f4bb32d787c444da8261b91891 is 50, key is test_row_0/A:col10/1730972196287/Put/seqid=0 2024-11-07T09:36:37,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742003_1179 (size=14794) 2024-11-07T09:36:37,451 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:37,456 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411079df2a4dd6c6e4fab891ccc186ffac383_81b216f4bb32d787c444da8261b91891 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411079df2a4dd6c6e4fab891ccc186ffac383_81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:37,457 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/a8202baf81214d1ba3dc374cc620fd19, store: [table=TestAcidGuarantees family=A region=81b216f4bb32d787c444da8261b91891] 2024-11-07T09:36:37,458 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/a8202baf81214d1ba3dc374cc620fd19 is 175, key is test_row_0/A:col10/1730972196287/Put/seqid=0 2024-11-07T09:36:37,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742004_1180 (size=39749) 2024-11-07T09:36:37,482 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=171, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/a8202baf81214d1ba3dc374cc620fd19 2024-11-07T09:36:37,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/0ca1a05a3d3d420b899158d593839555 is 50, key is test_row_0/B:col10/1730972196287/Put/seqid=0 2024-11-07T09:36:37,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:37,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972257493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:37,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:37,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:37,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:37,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:37,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972257495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:37,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:37,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:37,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972257495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:37,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:37,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:37,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:37,500 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:37,500 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:37,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972257495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:37,500 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:37,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:37,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:37,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:37,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742005_1181 (size=12151) 2024-11-07T09:36:37,506 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/0ca1a05a3d3d420b899158d593839555 2024-11-07T09:36:37,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/d1f07bb00cff4f50967dd31744082af7 is 50, key is test_row_0/C:col10/1730972196287/Put/seqid=0 2024-11-07T09:36:37,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742006_1182 (size=12151) 2024-11-07T09:36:37,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:37,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972257603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:37,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:37,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:37,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:37,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:37,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972257603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:37,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:37,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:37,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:37,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:37,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972257603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:37,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:37,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:37,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:37,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:37,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972257603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:37,606 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:37,606 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:37,606 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:37,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-07T09:36:37,624 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:36:37,624 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T09:36:37,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-07T09:36:37,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-07T09:36:37,626 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T09:36:37,627 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T09:36:37,627 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T09:36:37,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-07T09:36:37,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-07T09:36:37,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:37,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. as already flushing 2024-11-07T09:36:37,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:37,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:37,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:37,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:37,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:37,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972257813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:37,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:37,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:37,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972257813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:37,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:37,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:37,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:37,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972257813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:37,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:37,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:37,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:37,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972257814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:37,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:37,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:37,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:37,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:37,818 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:37,818 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:37,818 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:37,929 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/d1f07bb00cff4f50967dd31744082af7 2024-11-07T09:36:37,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/a8202baf81214d1ba3dc374cc620fd19 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/a8202baf81214d1ba3dc374cc620fd19 2024-11-07T09:36:37,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-07T09:36:37,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:37,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. as already flushing 2024-11-07T09:36:37,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:37,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:37,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:37,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:37,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-07T09:36:37,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/a8202baf81214d1ba3dc374cc620fd19, entries=200, sequenceid=171, filesize=38.8 K 2024-11-07T09:36:37,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/0ca1a05a3d3d420b899158d593839555 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/0ca1a05a3d3d420b899158d593839555 2024-11-07T09:36:37,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/0ca1a05a3d3d420b899158d593839555, entries=150, sequenceid=171, filesize=11.9 K 2024-11-07T09:36:37,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/d1f07bb00cff4f50967dd31744082af7 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/d1f07bb00cff4f50967dd31744082af7 2024-11-07T09:36:37,957 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/d1f07bb00cff4f50967dd31744082af7, entries=150, sequenceid=171, filesize=11.9 K 2024-11-07T09:36:37,961 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 81b216f4bb32d787c444da8261b91891 in 526ms, sequenceid=171, compaction requested=true 2024-11-07T09:36:37,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81b216f4bb32d787c444da8261b91891: 2024-11-07T09:36:37,961 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:36:37,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81b216f4bb32d787c444da8261b91891:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T09:36:37,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:37,961 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:36:37,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81b216f4bb32d787c444da8261b91891:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T09:36:37,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:37,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81b216f4bb32d787c444da8261b91891:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:36:37,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:36:37,963 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102183 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:36:37,963 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 81b216f4bb32d787c444da8261b91891/A is initiating minor compaction (all files) 2024-11-07T09:36:37,963 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:36:37,963 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81b216f4bb32d787c444da8261b91891/A in TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:37,963 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 81b216f4bb32d787c444da8261b91891/B is initiating minor compaction (all files) 2024-11-07T09:36:37,963 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81b216f4bb32d787c444da8261b91891/B in TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:37,963 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/8fc5b5bd41ff4409abb99c8777732f72, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/cdf7c281a874458598e364ea88f18867, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/a8202baf81214d1ba3dc374cc620fd19] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp, totalSize=99.8 K 2024-11-07T09:36:37,963 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:37,963 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/19b7869e32f84344a9c082a316a8479c, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/1cbde7ee907d4aff9751fe9f838874f3, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/0ca1a05a3d3d420b899158d593839555] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp, totalSize=35.8 K 2024-11-07T09:36:37,963 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. files: [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/8fc5b5bd41ff4409abb99c8777732f72, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/cdf7c281a874458598e364ea88f18867, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/a8202baf81214d1ba3dc374cc620fd19] 2024-11-07T09:36:37,964 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8fc5b5bd41ff4409abb99c8777732f72, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1730972192927 2024-11-07T09:36:37,964 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 19b7869e32f84344a9c082a316a8479c, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1730972192927 2024-11-07T09:36:37,964 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdf7c281a874458598e364ea88f18867, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1730972194106 2024-11-07T09:36:37,964 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cbde7ee907d4aff9751fe9f838874f3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1730972194106 2024-11-07T09:36:37,964 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ca1a05a3d3d420b899158d593839555, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1730972196287 2024-11-07T09:36:37,964 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8202baf81214d1ba3dc374cc620fd19, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1730972196287 2024-11-07T09:36:37,974 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=81b216f4bb32d787c444da8261b91891] 2024-11-07T09:36:37,975 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81b216f4bb32d787c444da8261b91891#B#compaction#157 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:37,976 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/5c2d7e23566b4b8488af3fb6f8f9aceb is 50, key is test_row_0/B:col10/1730972196287/Put/seqid=0 2024-11-07T09:36:37,978 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107b8ff3e078bed4faf88b875b8ffb01225_81b216f4bb32d787c444da8261b91891 store=[table=TestAcidGuarantees family=A region=81b216f4bb32d787c444da8261b91891] 2024-11-07T09:36:37,981 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107b8ff3e078bed4faf88b875b8ffb01225_81b216f4bb32d787c444da8261b91891, store=[table=TestAcidGuarantees family=A region=81b216f4bb32d787c444da8261b91891] 2024-11-07T09:36:37,982 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107b8ff3e078bed4faf88b875b8ffb01225_81b216f4bb32d787c444da8261b91891 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=81b216f4bb32d787c444da8261b91891] 2024-11-07T09:36:37,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742007_1183 (size=12527) 2024-11-07T09:36:37,992 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/5c2d7e23566b4b8488af3fb6f8f9aceb as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/5c2d7e23566b4b8488af3fb6f8f9aceb 2024-11-07T09:36:37,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742008_1184 (size=4469) 2024-11-07T09:36:37,999 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81b216f4bb32d787c444da8261b91891/B of 81b216f4bb32d787c444da8261b91891 into 5c2d7e23566b4b8488af3fb6f8f9aceb(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:37,999 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81b216f4bb32d787c444da8261b91891: 2024-11-07T09:36:37,999 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., storeName=81b216f4bb32d787c444da8261b91891/B, priority=13, startTime=1730972197961; duration=0sec 2024-11-07T09:36:37,999 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:36:37,999 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81b216f4bb32d787c444da8261b91891:B 2024-11-07T09:36:37,999 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:36:38,000 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:36:38,000 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 81b216f4bb32d787c444da8261b91891/C is initiating minor compaction (all files) 2024-11-07T09:36:38,001 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81b216f4bb32d787c444da8261b91891/C in TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:38,001 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/6df39d9e04834ea99ca6de2fc2c9f7b9, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/ed717274184c4321ad515ea731136aad, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/d1f07bb00cff4f50967dd31744082af7] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp, totalSize=35.8 K 2024-11-07T09:36:38,001 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 6df39d9e04834ea99ca6de2fc2c9f7b9, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1730972192927 2024-11-07T09:36:38,002 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting ed717274184c4321ad515ea731136aad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1730972194106 2024-11-07T09:36:38,002 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting d1f07bb00cff4f50967dd31744082af7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1730972196287 2024-11-07T09:36:38,011 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81b216f4bb32d787c444da8261b91891#C#compaction#158 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:38,011 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/59cc9f550b9240639206c48fa32961fd is 50, key is test_row_0/C:col10/1730972196287/Put/seqid=0 2024-11-07T09:36:38,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742009_1185 (size=12527) 2024-11-07T09:36:38,024 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/59cc9f550b9240639206c48fa32961fd as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/59cc9f550b9240639206c48fa32961fd 2024-11-07T09:36:38,030 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81b216f4bb32d787c444da8261b91891/C of 81b216f4bb32d787c444da8261b91891 into 59cc9f550b9240639206c48fa32961fd(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:38,030 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81b216f4bb32d787c444da8261b91891: 2024-11-07T09:36:38,030 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., storeName=81b216f4bb32d787c444da8261b91891/C, priority=13, startTime=1730972197961; duration=0sec 2024-11-07T09:36:38,030 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:38,030 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81b216f4bb32d787c444da8261b91891:C 2024-11-07T09:36:38,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-07T09:36:38,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:38,094 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 81b216f4bb32d787c444da8261b91891 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T09:36:38,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107531172a5ed28455297081ff50e6fd164_81b216f4bb32d787c444da8261b91891 is 50, key is test_row_0/A:col10/1730972197477/Put/seqid=0 2024-11-07T09:36:38,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742010_1186 (size=12304) 2024-11-07T09:36:38,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. as already flushing 2024-11-07T09:36:38,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:38,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:38,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972258131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:38,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:38,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:38,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:38,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:38,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972258136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:38,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:38,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:38,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:38,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:38,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:38,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972258137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:38,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972258137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:38,144 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:38,144 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:38,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:38,144 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:38,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:38,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:38,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:38,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972258243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:38,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:38,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:38,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:38,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-07T09:36:38,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:38,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972258253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:38,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:38,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:38,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:38,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:38,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972258253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:38,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:38,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:38,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:38,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:38,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972258253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:38,255 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:38,255 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:38,255 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:38,399 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81b216f4bb32d787c444da8261b91891#A#compaction#156 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:38,400 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/d442edf2fc87478abebcb331a41ed2ee is 175, key is test_row_0/A:col10/1730972196287/Put/seqid=0 2024-11-07T09:36:38,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742011_1187 (size=31481) 2024-11-07T09:36:38,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:38,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972258453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:38,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:38,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:38,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:38,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:38,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972258462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:38,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:38,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:38,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:38,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:38,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972258463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:38,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:38,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:38,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:38,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:38,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972258463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:38,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:38,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:38,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:38,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:38,532 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107531172a5ed28455297081ff50e6fd164_81b216f4bb32d787c444da8261b91891 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107531172a5ed28455297081ff50e6fd164_81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:38,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/b960794f3792449e8f5ef8a11cef8185, store: [table=TestAcidGuarantees family=A region=81b216f4bb32d787c444da8261b91891] 2024-11-07T09:36:38,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/b960794f3792449e8f5ef8a11cef8185 is 175, key is test_row_0/A:col10/1730972197477/Put/seqid=0 2024-11-07T09:36:38,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742012_1188 (size=31105) 2024-11-07T09:36:38,543 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=198, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/b960794f3792449e8f5ef8a11cef8185 2024-11-07T09:36:38,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/17fde19135994dfba93aa562331f120e is 50, key is test_row_0/B:col10/1730972197477/Put/seqid=0 2024-11-07T09:36:38,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742013_1189 (size=12151) 2024-11-07T09:36:38,611 INFO [master/9f1eb259a29b:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-07T09:36:38,612 INFO [master/9f1eb259a29b:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-07T09:36:38,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-07T09:36:38,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:38,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972258764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:38,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:38,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:38,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:38,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:38,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972258773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:38,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:38,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972258773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:38,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:38,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:38,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972258773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:38,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:38,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:38,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:38,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:38,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:38,774 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:38,774 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:38,774 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:38,812 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/d442edf2fc87478abebcb331a41ed2ee as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/d442edf2fc87478abebcb331a41ed2ee 2024-11-07T09:36:38,817 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81b216f4bb32d787c444da8261b91891/A of 81b216f4bb32d787c444da8261b91891 into d442edf2fc87478abebcb331a41ed2ee(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:38,817 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81b216f4bb32d787c444da8261b91891: 2024-11-07T09:36:38,817 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., storeName=81b216f4bb32d787c444da8261b91891/A, priority=13, startTime=1730972197961; duration=0sec 2024-11-07T09:36:38,817 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:38,817 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81b216f4bb32d787c444da8261b91891:A 2024-11-07T09:36:38,960 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/17fde19135994dfba93aa562331f120e 2024-11-07T09:36:38,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/d75a732dded94419b5a00ca04b511ddc is 50, key is test_row_0/C:col10/1730972197477/Put/seqid=0 2024-11-07T09:36:38,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742014_1190 (size=12151) 2024-11-07T09:36:39,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:39,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972259273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:39,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:39,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:39,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:39,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:39,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972259283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:39,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:39,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:39,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972259283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:39,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:39,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:39,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:39,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972259283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:39,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:39,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:39,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:39,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:39,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:39,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:39,373 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/d75a732dded94419b5a00ca04b511ddc 2024-11-07T09:36:39,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/b960794f3792449e8f5ef8a11cef8185 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/b960794f3792449e8f5ef8a11cef8185 2024-11-07T09:36:39,385 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/b960794f3792449e8f5ef8a11cef8185, entries=150, sequenceid=198, filesize=30.4 K 2024-11-07T09:36:39,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/17fde19135994dfba93aa562331f120e as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/17fde19135994dfba93aa562331f120e 2024-11-07T09:36:39,391 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/17fde19135994dfba93aa562331f120e, entries=150, sequenceid=198, filesize=11.9 K 2024-11-07T09:36:39,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/d75a732dded94419b5a00ca04b511ddc as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/d75a732dded94419b5a00ca04b511ddc 2024-11-07T09:36:39,398 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/d75a732dded94419b5a00ca04b511ddc, entries=150, sequenceid=198, filesize=11.9 K 2024-11-07T09:36:39,399 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 81b216f4bb32d787c444da8261b91891 in 1305ms, sequenceid=198, compaction requested=false 2024-11-07T09:36:39,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 81b216f4bb32d787c444da8261b91891: 2024-11-07T09:36:39,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:39,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-07T09:36:39,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-07T09:36:39,405 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-07T09:36:39,405 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7740 sec 2024-11-07T09:36:39,408 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.7820 sec 2024-11-07T09:36:39,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-07T09:36:39,773 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:36:39,774 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T09:36:39,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-07T09:36:39,776 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T09:36:39,777 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T09:36:39,777 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T09:36:39,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-07T09:36:39,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-07T09:36:39,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-07T09:36:39,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:39,930 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 81b216f4bb32d787c444da8261b91891 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T09:36:39,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107c9ad9c0732d449b2ad71d06ba94047a1_81b216f4bb32d787c444da8261b91891 is 50, key is test_row_0/A:col10/1730972198136/Put/seqid=0 2024-11-07T09:36:39,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742015_1191 (size=12304) 2024-11-07T09:36:39,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:39,951 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107c9ad9c0732d449b2ad71d06ba94047a1_81b216f4bb32d787c444da8261b91891 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107c9ad9c0732d449b2ad71d06ba94047a1_81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:39,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/5a0d0e5971424568a54ab43ab8e7beeb, store: [table=TestAcidGuarantees family=A region=81b216f4bb32d787c444da8261b91891] 2024-11-07T09:36:39,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/5a0d0e5971424568a54ab43ab8e7beeb is 175, key is test_row_0/A:col10/1730972198136/Put/seqid=0 2024-11-07T09:36:39,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742016_1192 (size=31105) 2024-11-07T09:36:40,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-07T09:36:40,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:40,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. as already flushing 2024-11-07T09:36:40,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:40,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972260320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:40,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:40,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:40,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:40,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972260321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:40,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:40,329 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:40,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:40,329 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:40,329 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:40,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972260327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:40,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:40,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:40,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:40,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:40,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972260329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:40,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:40,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:40,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:40,359 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=210, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/5a0d0e5971424568a54ab43ab8e7beeb 2024-11-07T09:36:40,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/06b79f12538340e2b1ea2b70cfb0ef05 is 50, key is test_row_0/B:col10/1730972198136/Put/seqid=0 2024-11-07T09:36:40,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742017_1193 (size=12151) 2024-11-07T09:36:40,389 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/06b79f12538340e2b1ea2b70cfb0ef05 2024-11-07T09:36:40,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/fce0e8511c3742bea0b58c0b4df4db52 is 50, key is test_row_0/C:col10/1730972198136/Put/seqid=0 2024-11-07T09:36:40,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-07T09:36:40,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742018_1194 (size=12151) 2024-11-07T09:36:40,419 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/fce0e8511c3742bea0b58c0b4df4db52 2024-11-07T09:36:40,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/5a0d0e5971424568a54ab43ab8e7beeb as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/5a0d0e5971424568a54ab43ab8e7beeb 2024-11-07T09:36:40,434 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/5a0d0e5971424568a54ab43ab8e7beeb, entries=150, sequenceid=210, filesize=30.4 K 2024-11-07T09:36:40,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/06b79f12538340e2b1ea2b70cfb0ef05 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/06b79f12538340e2b1ea2b70cfb0ef05 2024-11-07T09:36:40,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:40,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972260433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:40,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:40,440 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:40,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972260433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:40,441 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:40,441 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:40,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:40,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972260433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:40,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:40,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:40,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:40,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:40,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972260433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:40,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:40,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:40,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:40,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:40,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:40,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:40,444 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/06b79f12538340e2b1ea2b70cfb0ef05, entries=150, sequenceid=210, filesize=11.9 K 2024-11-07T09:36:40,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/fce0e8511c3742bea0b58c0b4df4db52 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/fce0e8511c3742bea0b58c0b4df4db52 2024-11-07T09:36:40,450 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/fce0e8511c3742bea0b58c0b4df4db52, entries=150, sequenceid=210, filesize=11.9 K 2024-11-07T09:36:40,451 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 81b216f4bb32d787c444da8261b91891 in 521ms, sequenceid=210, compaction requested=true 2024-11-07T09:36:40,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 81b216f4bb32d787c444da8261b91891: 2024-11-07T09:36:40,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:40,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-07T09:36:40,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-07T09:36:40,455 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-07T09:36:40,455 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 676 msec 2024-11-07T09:36:40,459 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 682 msec 2024-11-07T09:36:40,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:40,506 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:36:40,506 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81b216f4bb32d787c444da8261b91891 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-07T09:36:40,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107bfe87bb00bad4d46a690ee3ab08ec8ac_81b216f4bb32d787c444da8261b91891 is 50, key is test_row_0/A:col10/1730972200504/Put/seqid=0 2024-11-07T09:36:40,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:40,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1730972260536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:40,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:40,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:40,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:40,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742019_1195 (size=14794) 2024-11-07T09:36:40,553 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:40,558 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107bfe87bb00bad4d46a690ee3ab08ec8ac_81b216f4bb32d787c444da8261b91891 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107bfe87bb00bad4d46a690ee3ab08ec8ac_81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:40,560 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/a2e9291928d545098ea3c2de7a3c565b, store: [table=TestAcidGuarantees family=A region=81b216f4bb32d787c444da8261b91891] 2024-11-07T09:36:40,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/a2e9291928d545098ea3c2de7a3c565b is 175, key is test_row_0/A:col10/1730972200504/Put/seqid=0 2024-11-07T09:36:40,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742020_1196 (size=39749) 2024-11-07T09:36:40,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:40,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972260643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:40,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:40,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972260643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:40,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:40,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:40,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:40,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:40,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1730972260643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:40,649 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:40,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:40,649 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:40,649 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:40,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:40,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:40,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:40,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972260652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:40,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:40,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:40,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:40,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:40,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972260653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:40,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:40,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:40,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:40,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:40,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1730972260853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:40,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:40,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:40,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:40,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-07T09:36:40,913 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:36:40,914 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T09:36:40,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-07T09:36:40,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-07T09:36:40,916 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T09:36:40,917 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T09:36:40,917 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T09:36:40,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:40,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972260953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:40,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:40,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972260953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:40,958 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:40,958 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:40,958 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:40,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:40,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:40,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:40,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:40,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972260963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:40,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:40,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:40,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972260963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:40,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:40,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:40,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:40,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:40,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:40,978 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=236, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/a2e9291928d545098ea3c2de7a3c565b 2024-11-07T09:36:40,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/6ad46c6bd12940b4a51b6fcb849dd523 is 50, key is test_row_0/B:col10/1730972200504/Put/seqid=0 2024-11-07T09:36:40,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742021_1197 (size=12151) 2024-11-07T09:36:41,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-07T09:36:41,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-07T09:36:41,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:41,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. as already flushing 2024-11-07T09:36:41,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:41,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:41,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:41,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:41,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:41,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1730972261163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:41,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:41,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:41,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:41,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-07T09:36:41,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:41,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. as already flushing 2024-11-07T09:36:41,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:41,224 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:41,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:41,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:41,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-07T09:36:41,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-07T09:36:41,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:41,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. as already flushing 2024-11-07T09:36:41,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:41,377 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:41,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:41,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:41,395 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/6ad46c6bd12940b4a51b6fcb849dd523 2024-11-07T09:36:41,405 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/6bb0d45573884ef9a4c75dfc6d8f7e5d is 50, key is test_row_0/C:col10/1730972200504/Put/seqid=0 2024-11-07T09:36:41,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742022_1198 (size=12151) 2024-11-07T09:36:41,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:41,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972261463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:41,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:41,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972261463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:41,470 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:41,470 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:41,470 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:41,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:41,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:41,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:41,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:41,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972261472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:41,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:41,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:41,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:41,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:41,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972261472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:41,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:41,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:41,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:41,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-07T09:36:41,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:41,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. as already flushing 2024-11-07T09:36:41,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:41,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:41,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:41,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:41,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-07T09:36:41,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:41,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1730972261673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:41,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:41,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:41,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:41,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-07T09:36:41,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:41,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. as already flushing 2024-11-07T09:36:41,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:41,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:41,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:41,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:41,810 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/6bb0d45573884ef9a4c75dfc6d8f7e5d 2024-11-07T09:36:41,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/a2e9291928d545098ea3c2de7a3c565b as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/a2e9291928d545098ea3c2de7a3c565b 2024-11-07T09:36:41,821 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/a2e9291928d545098ea3c2de7a3c565b, entries=200, sequenceid=236, filesize=38.8 K 2024-11-07T09:36:41,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/6ad46c6bd12940b4a51b6fcb849dd523 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/6ad46c6bd12940b4a51b6fcb849dd523 2024-11-07T09:36:41,830 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/6ad46c6bd12940b4a51b6fcb849dd523, entries=150, sequenceid=236, filesize=11.9 K 2024-11-07T09:36:41,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/6bb0d45573884ef9a4c75dfc6d8f7e5d as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/6bb0d45573884ef9a4c75dfc6d8f7e5d 2024-11-07T09:36:41,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-07T09:36:41,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:41,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. as already flushing 2024-11-07T09:36:41,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:41,838 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/6bb0d45573884ef9a4c75dfc6d8f7e5d, entries=150, sequenceid=236, filesize=11.9 K 2024-11-07T09:36:41,838 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:41,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:41,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:41,840 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 81b216f4bb32d787c444da8261b91891 in 1334ms, sequenceid=236, compaction requested=true 2024-11-07T09:36:41,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81b216f4bb32d787c444da8261b91891: 2024-11-07T09:36:41,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81b216f4bb32d787c444da8261b91891:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T09:36:41,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:41,840 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T09:36:41,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81b216f4bb32d787c444da8261b91891:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T09:36:41,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:41,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81b216f4bb32d787c444da8261b91891:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:36:41,840 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T09:36:41,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:36:41,844 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133440 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T09:36:41,844 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 81b216f4bb32d787c444da8261b91891/A is initiating minor compaction (all files) 2024-11-07T09:36:41,844 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81b216f4bb32d787c444da8261b91891/A in TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:41,844 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/d442edf2fc87478abebcb331a41ed2ee, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/b960794f3792449e8f5ef8a11cef8185, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/5a0d0e5971424568a54ab43ab8e7beeb, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/a2e9291928d545098ea3c2de7a3c565b] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp, totalSize=130.3 K 2024-11-07T09:36:41,844 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:41,844 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. files: [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/d442edf2fc87478abebcb331a41ed2ee, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/b960794f3792449e8f5ef8a11cef8185, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/5a0d0e5971424568a54ab43ab8e7beeb, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/a2e9291928d545098ea3c2de7a3c565b] 2024-11-07T09:36:41,845 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T09:36:41,845 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 81b216f4bb32d787c444da8261b91891/B is initiating minor compaction (all files) 2024-11-07T09:36:41,845 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81b216f4bb32d787c444da8261b91891/B in TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:41,845 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/5c2d7e23566b4b8488af3fb6f8f9aceb, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/17fde19135994dfba93aa562331f120e, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/06b79f12538340e2b1ea2b70cfb0ef05, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/6ad46c6bd12940b4a51b6fcb849dd523] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp, totalSize=47.8 K 2024-11-07T09:36:41,845 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c2d7e23566b4b8488af3fb6f8f9aceb, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1730972196287 2024-11-07T09:36:41,845 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting d442edf2fc87478abebcb331a41ed2ee, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1730972196287 2024-11-07T09:36:41,846 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 17fde19135994dfba93aa562331f120e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1730972197477 2024-11-07T09:36:41,846 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting b960794f3792449e8f5ef8a11cef8185, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1730972197477 2024-11-07T09:36:41,847 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 06b79f12538340e2b1ea2b70cfb0ef05, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1730972198128 2024-11-07T09:36:41,847 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a0d0e5971424568a54ab43ab8e7beeb, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1730972198128 2024-11-07T09:36:41,847 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ad46c6bd12940b4a51b6fcb849dd523, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1730972200319 2024-11-07T09:36:41,847 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2e9291928d545098ea3c2de7a3c565b, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1730972200318 2024-11-07T09:36:41,863 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=81b216f4bb32d787c444da8261b91891] 2024-11-07T09:36:41,866 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81b216f4bb32d787c444da8261b91891#B#compaction#168 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:41,867 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/a28dd263201f4b97bf76dd7130c4e730 is 50, key is test_row_0/B:col10/1730972200504/Put/seqid=0 2024-11-07T09:36:41,889 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411076a6794cc23854004bbefbe7db7702f01_81b216f4bb32d787c444da8261b91891 store=[table=TestAcidGuarantees family=A region=81b216f4bb32d787c444da8261b91891] 2024-11-07T09:36:41,891 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411076a6794cc23854004bbefbe7db7702f01_81b216f4bb32d787c444da8261b91891, store=[table=TestAcidGuarantees family=A region=81b216f4bb32d787c444da8261b91891] 2024-11-07T09:36:41,891 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411076a6794cc23854004bbefbe7db7702f01_81b216f4bb32d787c444da8261b91891 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=81b216f4bb32d787c444da8261b91891] 2024-11-07T09:36:41,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742023_1199 (size=12663) 2024-11-07T09:36:41,906 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/a28dd263201f4b97bf76dd7130c4e730 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/a28dd263201f4b97bf76dd7130c4e730 2024-11-07T09:36:41,919 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 81b216f4bb32d787c444da8261b91891/B of 81b216f4bb32d787c444da8261b91891 into a28dd263201f4b97bf76dd7130c4e730(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:41,920 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81b216f4bb32d787c444da8261b91891: 2024-11-07T09:36:41,920 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., storeName=81b216f4bb32d787c444da8261b91891/B, priority=12, startTime=1730972201840; duration=0sec 2024-11-07T09:36:41,920 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:36:41,920 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81b216f4bb32d787c444da8261b91891:B 2024-11-07T09:36:41,920 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T09:36:41,922 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T09:36:41,922 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 81b216f4bb32d787c444da8261b91891/C is initiating minor compaction (all files) 2024-11-07T09:36:41,922 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81b216f4bb32d787c444da8261b91891/C in TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:41,922 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/59cc9f550b9240639206c48fa32961fd, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/d75a732dded94419b5a00ca04b511ddc, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/fce0e8511c3742bea0b58c0b4df4db52, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/6bb0d45573884ef9a4c75dfc6d8f7e5d] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp, totalSize=47.8 K 2024-11-07T09:36:41,923 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 59cc9f550b9240639206c48fa32961fd, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1730972196287 2024-11-07T09:36:41,923 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting d75a732dded94419b5a00ca04b511ddc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1730972197477 2024-11-07T09:36:41,923 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting fce0e8511c3742bea0b58c0b4df4db52, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1730972198128 2024-11-07T09:36:41,924 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bb0d45573884ef9a4c75dfc6d8f7e5d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1730972200319 2024-11-07T09:36:41,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742024_1200 (size=4469) 2024-11-07T09:36:41,935 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81b216f4bb32d787c444da8261b91891#A#compaction#169 average throughput is 0.34 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:41,936 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/0d2ef765f0ac4bb2b46fd92cbe10816d is 175, key is test_row_0/A:col10/1730972200504/Put/seqid=0 2024-11-07T09:36:41,942 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81b216f4bb32d787c444da8261b91891#C#compaction#170 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:41,943 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/58e613423a2040e8b9677bdd390f31e5 is 50, key is test_row_0/C:col10/1730972200504/Put/seqid=0 2024-11-07T09:36:41,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742025_1201 (size=31617) 2024-11-07T09:36:41,963 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/0d2ef765f0ac4bb2b46fd92cbe10816d as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/0d2ef765f0ac4bb2b46fd92cbe10816d 2024-11-07T09:36:41,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742026_1202 (size=12663) 2024-11-07T09:36:41,973 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 81b216f4bb32d787c444da8261b91891/A of 81b216f4bb32d787c444da8261b91891 into 0d2ef765f0ac4bb2b46fd92cbe10816d(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:41,973 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81b216f4bb32d787c444da8261b91891: 2024-11-07T09:36:41,973 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., storeName=81b216f4bb32d787c444da8261b91891/A, priority=12, startTime=1730972201840; duration=0sec 2024-11-07T09:36:41,973 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:41,973 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81b216f4bb32d787c444da8261b91891:A 2024-11-07T09:36:41,975 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/58e613423a2040e8b9677bdd390f31e5 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/58e613423a2040e8b9677bdd390f31e5 2024-11-07T09:36:41,984 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 81b216f4bb32d787c444da8261b91891/C of 81b216f4bb32d787c444da8261b91891 into 58e613423a2040e8b9677bdd390f31e5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:41,984 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81b216f4bb32d787c444da8261b91891: 2024-11-07T09:36:41,984 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., storeName=81b216f4bb32d787c444da8261b91891/C, priority=12, startTime=1730972201840; duration=0sec 2024-11-07T09:36:41,984 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:41,984 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81b216f4bb32d787c444da8261b91891:C 2024-11-07T09:36:41,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-07T09:36:41,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:41,994 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 81b216f4bb32d787c444da8261b91891 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-07T09:36:42,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107c0ad0cd2b4894bbc9d105ad91abe4e38_81b216f4bb32d787c444da8261b91891 is 50, key is test_row_0/A:col10/1730972200517/Put/seqid=0 2024-11-07T09:36:42,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742027_1203 (size=12304) 2024-11-07T09:36:42,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:42,045 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107c0ad0cd2b4894bbc9d105ad91abe4e38_81b216f4bb32d787c444da8261b91891 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107c0ad0cd2b4894bbc9d105ad91abe4e38_81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:42,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/6ef5ab0c44cb474daf3c8fcb782722d2, store: [table=TestAcidGuarantees family=A region=81b216f4bb32d787c444da8261b91891] 2024-11-07T09:36:42,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/6ef5ab0c44cb474daf3c8fcb782722d2 is 175, key is test_row_0/A:col10/1730972200517/Put/seqid=0 2024-11-07T09:36:42,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742028_1204 (size=31105) 2024-11-07T09:36:42,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-07T09:36:42,054 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=249, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/6ef5ab0c44cb474daf3c8fcb782722d2 2024-11-07T09:36:42,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/97caa18626f4400cb0a95b5d76682cf5 is 50, key is test_row_0/B:col10/1730972200517/Put/seqid=0 2024-11-07T09:36:42,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742029_1205 (size=12151) 2024-11-07T09:36:42,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:42,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. as already flushing 2024-11-07T09:36:42,496 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/97caa18626f4400cb0a95b5d76682cf5 2024-11-07T09:36:42,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/cea1ffdc225f4259801927af4c836705 is 50, key is test_row_0/C:col10/1730972200517/Put/seqid=0 2024-11-07T09:36:42,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742030_1206 (size=12151) 2024-11-07T09:36:42,515 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/cea1ffdc225f4259801927af4c836705 2024-11-07T09:36:42,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:42,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972262519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:42,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:42,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:42,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:42,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/6ef5ab0c44cb474daf3c8fcb782722d2 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/6ef5ab0c44cb474daf3c8fcb782722d2 2024-11-07T09:36:42,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:42,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972262519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:42,526 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:42,526 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:42,526 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:42,527 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/6ef5ab0c44cb474daf3c8fcb782722d2, entries=150, sequenceid=249, filesize=30.4 K 2024-11-07T09:36:42,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/97caa18626f4400cb0a95b5d76682cf5 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/97caa18626f4400cb0a95b5d76682cf5 2024-11-07T09:36:42,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:42,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972262521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:42,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:42,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972262521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:42,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:42,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:42,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:42,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:42,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:42,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:42,533 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/97caa18626f4400cb0a95b5d76682cf5, entries=150, sequenceid=249, filesize=11.9 K 2024-11-07T09:36:42,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/cea1ffdc225f4259801927af4c836705 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/cea1ffdc225f4259801927af4c836705 2024-11-07T09:36:42,539 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/cea1ffdc225f4259801927af4c836705, entries=150, sequenceid=249, filesize=11.9 K 2024-11-07T09:36:42,540 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 81b216f4bb32d787c444da8261b91891 in 546ms, sequenceid=249, compaction requested=false 2024-11-07T09:36:42,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 81b216f4bb32d787c444da8261b91891: 2024-11-07T09:36:42,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:42,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-07T09:36:42,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-07T09:36:42,544 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-07T09:36:42,544 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6250 sec 2024-11-07T09:36:42,546 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.6310 sec 2024-11-07T09:36:42,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:42,636 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:36:42,636 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81b216f4bb32d787c444da8261b91891 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-07T09:36:42,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:42,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972262644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:42,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:42,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:42,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:42,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972262644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:42,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:42,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:42,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:42,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972262645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:42,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:42,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:42,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:42,649 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:42,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972262645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:42,649 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:42,649 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:42,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:42,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:42,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:42,660 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110770ede35b35b04f9facd1100dd0b75010_81b216f4bb32d787c444da8261b91891 is 50, key is test_row_0/A:col10/1730972202635/Put/seqid=0 2024-11-07T09:36:42,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742031_1207 (size=20074) 2024-11-07T09:36:42,664 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:42,668 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110770ede35b35b04f9facd1100dd0b75010_81b216f4bb32d787c444da8261b91891 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110770ede35b35b04f9facd1100dd0b75010_81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:42,670 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/85a324bf2ba6403aa99be47bbb568062, store: [table=TestAcidGuarantees family=A region=81b216f4bb32d787c444da8261b91891] 2024-11-07T09:36:42,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/85a324bf2ba6403aa99be47bbb568062 is 175, key is test_row_0/A:col10/1730972202635/Put/seqid=0 2024-11-07T09:36:42,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742032_1208 (size=57333) 2024-11-07T09:36:42,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:42,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1730972262682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:42,685 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 9f1eb259a29b,42895,1730972153970 for 'test_row_2' in 81b216f4bb32d787c444da8261b91891 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2150 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T09:36:42,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:42,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:42,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:42,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:42,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972262753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:42,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:42,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:42,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:42,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:42,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972262754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:42,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:42,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:42,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:42,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:42,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972262755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:42,760 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:42,760 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:42,760 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:42,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:42,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972262755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:42,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:42,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:42,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:42,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:42,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972262963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:42,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:42,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:42,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:42,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:42,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972262963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:42,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:42,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:42,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:42,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:42,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972262964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:42,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:42,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972262964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:42,966 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:42,966 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:42,966 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:42,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:42,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:42,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:43,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-07T09:36:43,063 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:36:43,063 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T09:36:43,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-07T09:36:43,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-07T09:36:43,065 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T09:36:43,066 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T09:36:43,066 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T09:36:43,079 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=278, memsize=58.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/85a324bf2ba6403aa99be47bbb568062 2024-11-07T09:36:43,087 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/cd7f6ed3b7ce45c2b6f9a8d8d44ad738 is 50, key is test_row_0/B:col10/1730972202635/Put/seqid=0 2024-11-07T09:36:43,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742033_1209 (size=12301) 2024-11-07T09:36:43,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-07T09:36:43,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-07T09:36:43,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:43,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. as already flushing 2024-11-07T09:36:43,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:43,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:43,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:43,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:43,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:43,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:43,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972263273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:43,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972263273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:43,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:43,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:43,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:43,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:43,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972263274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:43,275 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:43,275 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:43,275 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:43,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:43,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:43,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:43,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:43,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972263274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:43,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:43,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:43,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:43,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-07T09:36:43,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:43,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. as already flushing 2024-11-07T09:36:43,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:43,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:43,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:43,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:43,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-07T09:36:43,493 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/cd7f6ed3b7ce45c2b6f9a8d8d44ad738 2024-11-07T09:36:43,504 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/408df9aeaae54fd6a6799c65415a408b is 50, key is test_row_0/C:col10/1730972202635/Put/seqid=0 2024-11-07T09:36:43,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742034_1210 (size=12301) 2024-11-07T09:36:43,509 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/408df9aeaae54fd6a6799c65415a408b 2024-11-07T09:36:43,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/85a324bf2ba6403aa99be47bbb568062 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/85a324bf2ba6403aa99be47bbb568062 2024-11-07T09:36:43,518 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/85a324bf2ba6403aa99be47bbb568062, entries=300, sequenceid=278, filesize=56.0 K 2024-11-07T09:36:43,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/cd7f6ed3b7ce45c2b6f9a8d8d44ad738 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/cd7f6ed3b7ce45c2b6f9a8d8d44ad738 2024-11-07T09:36:43,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-07T09:36:43,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:43,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. as already flushing 2024-11-07T09:36:43,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:43,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/cd7f6ed3b7ce45c2b6f9a8d8d44ad738, entries=150, sequenceid=278, filesize=12.0 K 2024-11-07T09:36:43,526 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:43,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:43,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:43,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/408df9aeaae54fd6a6799c65415a408b as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/408df9aeaae54fd6a6799c65415a408b 2024-11-07T09:36:43,532 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/408df9aeaae54fd6a6799c65415a408b, entries=150, sequenceid=278, filesize=12.0 K 2024-11-07T09:36:43,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,540 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=33.54 KB/34350 for 81b216f4bb32d787c444da8261b91891 in 904ms, sequenceid=278, compaction requested=true 2024-11-07T09:36:43,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81b216f4bb32d787c444da8261b91891: 2024-11-07T09:36:43,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81b216f4bb32d787c444da8261b91891:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T09:36:43,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:43,540 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:36:43,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81b216f4bb32d787c444da8261b91891:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T09:36:43,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:36:43,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81b216f4bb32d787c444da8261b91891:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:36:43,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T09:36:43,541 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:36:43,542 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:36:43,542 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 81b216f4bb32d787c444da8261b91891/B is initiating minor compaction (all files) 2024-11-07T09:36:43,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,542 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81b216f4bb32d787c444da8261b91891/B in TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:43,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,542 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/a28dd263201f4b97bf76dd7130c4e730, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/97caa18626f4400cb0a95b5d76682cf5, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/cd7f6ed3b7ce45c2b6f9a8d8d44ad738] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp, totalSize=36.2 K 2024-11-07T09:36:43,543 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 120055 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:36:43,543 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 81b216f4bb32d787c444da8261b91891/A is initiating minor compaction (all files) 2024-11-07T09:36:43,543 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81b216f4bb32d787c444da8261b91891/A in TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:43,543 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/0d2ef765f0ac4bb2b46fd92cbe10816d, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/6ef5ab0c44cb474daf3c8fcb782722d2, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/85a324bf2ba6403aa99be47bbb568062] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp, totalSize=117.2 K 2024-11-07T09:36:43,543 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:43,543 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. files: [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/0d2ef765f0ac4bb2b46fd92cbe10816d, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/6ef5ab0c44cb474daf3c8fcb782722d2, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/85a324bf2ba6403aa99be47bbb568062] 2024-11-07T09:36:43,543 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting a28dd263201f4b97bf76dd7130c4e730, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1730972200319 2024-11-07T09:36:43,544 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d2ef765f0ac4bb2b46fd92cbe10816d, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1730972200319 2024-11-07T09:36:43,544 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97caa18626f4400cb0a95b5d76682cf5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1730972200517 2024-11-07T09:36:43,544 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ef5ab0c44cb474daf3c8fcb782722d2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1730972200517 2024-11-07T09:36:43,544 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd7f6ed3b7ce45c2b6f9a8d8d44ad738, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1730972202520 2024-11-07T09:36:43,544 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 85a324bf2ba6403aa99be47bbb568062, keycount=300, bloomtype=ROW, size=56.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1730972202517 2024-11-07T09:36:43,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,553 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=81b216f4bb32d787c444da8261b91891] 2024-11-07T09:36:43,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,558 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81b216f4bb32d787c444da8261b91891#B#compaction#178 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:43,559 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/a46b734b757b479099e69792d028cfd6 is 50, key is test_row_0/B:col10/1730972202635/Put/seqid=0 2024-11-07T09:36:43,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,562 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411074ce9bb8aa05f4bdc97fd0ba95cc48859_81b216f4bb32d787c444da8261b91891 store=[table=TestAcidGuarantees family=A region=81b216f4bb32d787c444da8261b91891] 2024-11-07T09:36:43,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,570 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411074ce9bb8aa05f4bdc97fd0ba95cc48859_81b216f4bb32d787c444da8261b91891, store=[table=TestAcidGuarantees family=A region=81b216f4bb32d787c444da8261b91891] 2024-11-07T09:36:43,570 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411074ce9bb8aa05f4bdc97fd0ba95cc48859_81b216f4bb32d787c444da8261b91891 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=81b216f4bb32d787c444da8261b91891] 2024-11-07T09:36:43,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742035_1211 (size=12915) 2024-11-07T09:36:43,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742036_1212 (size=4469) 2024-11-07T09:36:43,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,623 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81b216f4bb32d787c444da8261b91891#A#compaction#177 average throughput is 0.35 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:43,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,624 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/3a6716d59a4946f2976e15f1e238a9cd is 175, key is test_row_0/A:col10/1730972202635/Put/seqid=0 2024-11-07T09:36:43,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742037_1213 (size=31869) 2024-11-07T09:36:43,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,636 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/3a6716d59a4946f2976e15f1e238a9cd as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/3a6716d59a4946f2976e15f1e238a9cd 2024-11-07T09:36:43,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,644 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81b216f4bb32d787c444da8261b91891/A of 81b216f4bb32d787c444da8261b91891 into 3a6716d59a4946f2976e15f1e238a9cd(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:43,644 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81b216f4bb32d787c444da8261b91891: 2024-11-07T09:36:43,644 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., storeName=81b216f4bb32d787c444da8261b91891/A, priority=13, startTime=1730972203540; duration=0sec 2024-11-07T09:36:43,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,644 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:36:43,644 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81b216f4bb32d787c444da8261b91891:A 2024-11-07T09:36:43,644 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:36:43,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,646 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:36:43,646 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 81b216f4bb32d787c444da8261b91891/C is initiating minor compaction (all files) 2024-11-07T09:36:43,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,646 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81b216f4bb32d787c444da8261b91891/C in TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:43,646 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/58e613423a2040e8b9677bdd390f31e5, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/cea1ffdc225f4259801927af4c836705, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/408df9aeaae54fd6a6799c65415a408b] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp, totalSize=36.2 K 2024-11-07T09:36:43,647 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 58e613423a2040e8b9677bdd390f31e5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1730972200319 2024-11-07T09:36:43,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,648 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting cea1ffdc225f4259801927af4c836705, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1730972200517 2024-11-07T09:36:43,649 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 408df9aeaae54fd6a6799c65415a408b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1730972202520 2024-11-07T09:36:43,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,664 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81b216f4bb32d787c444da8261b91891#C#compaction#179 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:43,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,665 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/cf2bd8fc1343428494f209991176bc61 is 50, key is test_row_0/C:col10/1730972202635/Put/seqid=0 2024-11-07T09:36:43,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-07T09:36:43,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:43,680 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 81b216f4bb32d787c444da8261b91891 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-07T09:36:43,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742038_1214 (size=12915) 2024-11-07T09:36:43,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-07T09:36:43,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,694 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/cf2bd8fc1343428494f209991176bc61 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/cf2bd8fc1343428494f209991176bc61 2024-11-07T09:36:43,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,700 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81b216f4bb32d787c444da8261b91891/C of 81b216f4bb32d787c444da8261b91891 into cf2bd8fc1343428494f209991176bc61(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:43,700 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81b216f4bb32d787c444da8261b91891: 2024-11-07T09:36:43,700 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., storeName=81b216f4bb32d787c444da8261b91891/C, priority=13, startTime=1730972203540; duration=0sec 2024-11-07T09:36:43,700 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:43,700 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81b216f4bb32d787c444da8261b91891:C 2024-11-07T09:36:43,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107c5f6f095f5a94831966feb96b666ad0c_81b216f4bb32d787c444da8261b91891 is 50, key is test_row_0/A:col10/1730972202643/Put/seqid=0 2024-11-07T09:36:43,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742039_1215 (size=9914) 2024-11-07T09:36:43,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,747 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107c5f6f095f5a94831966feb96b666ad0c_81b216f4bb32d787c444da8261b91891 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107c5f6f095f5a94831966feb96b666ad0c_81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:43,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/3144bb6601124f0192cfb04c629890b5, store: [table=TestAcidGuarantees family=A region=81b216f4bb32d787c444da8261b91891] 2024-11-07T09:36:43,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/3144bb6601124f0192cfb04c629890b5 is 175, key is test_row_0/A:col10/1730972202643/Put/seqid=0 2024-11-07T09:36:43,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742040_1216 (size=22561) 2024-11-07T09:36:43,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:43,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. as already flushing 2024-11-07T09:36:43,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:43,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:43,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972263913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:43,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:43,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972263914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:43,919 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:43,919 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:43,920 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:43,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:43,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:43,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:43,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:43,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972263918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:43,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:43,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972263919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:43,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:43,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:43,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:43,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:43,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:43,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:43,993 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/a46b734b757b479099e69792d028cfd6 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/a46b734b757b479099e69792d028cfd6 2024-11-07T09:36:43,998 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81b216f4bb32d787c444da8261b91891/B of 81b216f4bb32d787c444da8261b91891 into a46b734b757b479099e69792d028cfd6(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:43,998 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81b216f4bb32d787c444da8261b91891: 2024-11-07T09:36:43,998 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., storeName=81b216f4bb32d787c444da8261b91891/B, priority=13, startTime=1730972203540; duration=0sec 2024-11-07T09:36:43,998 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:43,998 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81b216f4bb32d787c444da8261b91891:B 2024-11-07T09:36:44,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:44,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972264023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:44,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:44,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972264023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:44,026 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:44,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:44,026 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:44,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:44,026 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:44,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:44,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:44,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972264033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:44,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:44,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:44,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:44,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972264034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:44,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:44,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:44,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:44,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:44,158 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=287, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/3144bb6601124f0192cfb04c629890b5 2024-11-07T09:36:44,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/b56c79bca4324917a98823c47afb4c7e is 50, key is test_row_0/B:col10/1730972202643/Put/seqid=0 2024-11-07T09:36:44,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742041_1217 (size=9857) 2024-11-07T09:36:44,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-07T09:36:44,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:44,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972264233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:44,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:44,234 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:44,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972264233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:44,234 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:44,234 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:44,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:44,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:44,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:44,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:44,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972264243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:44,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:44,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972264243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:44,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:44,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:44,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:44,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:44,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:44,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:44,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:44,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:44,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972264543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:44,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972264543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:44,546 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:44,546 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:44,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:44,546 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:44,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:44,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:44,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:44,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972264553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:44,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:44,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:44,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:44,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:44,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972264554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:44,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:44,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:44,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:44,580 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/b56c79bca4324917a98823c47afb4c7e 2024-11-07T09:36:44,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/98b5a61ea61b483e88d068e4f148e0e3 is 50, key is test_row_0/C:col10/1730972202643/Put/seqid=0 2024-11-07T09:36:44,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742042_1218 (size=9857) 2024-11-07T09:36:44,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:44,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1730972264705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:44,709 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 9f1eb259a29b,42895,1730972153970 for 'test_row_2' in 81b216f4bb32d787c444da8261b91891 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T09:36:44,709 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:44,709 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:44,709 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:45,002 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/98b5a61ea61b483e88d068e4f148e0e3 2024-11-07T09:36:45,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/3144bb6601124f0192cfb04c629890b5 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/3144bb6601124f0192cfb04c629890b5 2024-11-07T09:36:45,013 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/3144bb6601124f0192cfb04c629890b5, entries=100, sequenceid=287, filesize=22.0 K 2024-11-07T09:36:45,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/b56c79bca4324917a98823c47afb4c7e as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/b56c79bca4324917a98823c47afb4c7e 2024-11-07T09:36:45,020 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/b56c79bca4324917a98823c47afb4c7e, entries=100, sequenceid=287, filesize=9.6 K 2024-11-07T09:36:45,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/98b5a61ea61b483e88d068e4f148e0e3 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/98b5a61ea61b483e88d068e4f148e0e3 2024-11-07T09:36:45,028 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/98b5a61ea61b483e88d068e4f148e0e3, entries=100, sequenceid=287, filesize=9.6 K 2024-11-07T09:36:45,030 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for 81b216f4bb32d787c444da8261b91891 in 1350ms, sequenceid=287, compaction requested=false 2024-11-07T09:36:45,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 81b216f4bb32d787c444da8261b91891: 2024-11-07T09:36:45,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:45,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-07T09:36:45,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-07T09:36:45,034 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-07T09:36:45,034 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9660 sec 2024-11-07T09:36:45,037 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 1.9720 sec 2024-11-07T09:36:45,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:45,054 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:36:45,054 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81b216f4bb32d787c444da8261b91891 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-07T09:36:45,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:45,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:45,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972265060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:45,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972265059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:45,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107cb64dc0201894a25b67c7d9e961b120e_81b216f4bb32d787c444da8261b91891 is 50, key is test_row_0/A:col10/1730972205054/Put/seqid=0 2024-11-07T09:36:45,065 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:45,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:45,065 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:45,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:45,065 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:45,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:45,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:45,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972265063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:45,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:45,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:45,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972265065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:45,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:45,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:45,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:45,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:45,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:45,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742043_1219 (size=14994) 2024-11-07T09:36:45,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:45,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972265173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:45,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:45,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:45,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:45,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:45,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972265173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:45,177 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:45,177 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:45,177 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:45,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-07T09:36:45,213 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:36:45,214 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T09:36:45,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-07T09:36:45,216 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T09:36:45,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-07T09:36:45,216 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T09:36:45,216 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T09:36:45,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-07T09:36:45,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-07T09:36:45,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:45,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. as already flushing 2024-11-07T09:36:45,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:45,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:45,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:45,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:45,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:45,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:45,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972265383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:45,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972265383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:45,387 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:45,387 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:45,387 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:45,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:45,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:45,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:45,479 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,485 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107cb64dc0201894a25b67c7d9e961b120e_81b216f4bb32d787c444da8261b91891 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107cb64dc0201894a25b67c7d9e961b120e_81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:45,489 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/f6b59c7ad90a4f2281529ad52ebe4aca, store: [table=TestAcidGuarantees family=A region=81b216f4bb32d787c444da8261b91891] 2024-11-07T09:36:45,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/f6b59c7ad90a4f2281529ad52ebe4aca is 175, key is test_row_0/A:col10/1730972205054/Put/seqid=0 2024-11-07T09:36:45,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742044_1220 (size=39949) 2024-11-07T09:36:45,501 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=319, memsize=60.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/f6b59c7ad90a4f2281529ad52ebe4aca 2024-11-07T09:36:45,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/ddf6b87ea3d44cd3874b541c057e900b is 50, key is test_row_0/B:col10/1730972205054/Put/seqid=0 2024-11-07T09:36:45,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742045_1221 (size=12301) 2024-11-07T09:36:45,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-07T09:36:45,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:45,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. as already flushing 2024-11-07T09:36:45,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:45,522 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:45,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:45,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:45,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-07T09:36:45,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-07T09:36:45,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:45,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. as already flushing 2024-11-07T09:36:45,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:45,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:45,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:45,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:45,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:45,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972265693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:45,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:45,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:45,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:45,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972265693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:45,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:45,694 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:45,694 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:45,694 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:45,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-07T09:36:45,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:45,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. as already flushing 2024-11-07T09:36:45,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:45,830 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:45,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:45,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:45,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-07T09:36:45,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/ddf6b87ea3d44cd3874b541c057e900b 2024-11-07T09:36:45,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/9b24b643d93a4b30b986c7e444ceb70a is 50, key is test_row_0/C:col10/1730972205054/Put/seqid=0 2024-11-07T09:36:45,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742046_1222 (size=12301) 2024-11-07T09:36:45,938 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/9b24b643d93a4b30b986c7e444ceb70a 2024-11-07T09:36:45,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/f6b59c7ad90a4f2281529ad52ebe4aca as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/f6b59c7ad90a4f2281529ad52ebe4aca 2024-11-07T09:36:45,950 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/f6b59c7ad90a4f2281529ad52ebe4aca, entries=200, sequenceid=319, filesize=39.0 K 2024-11-07T09:36:45,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/ddf6b87ea3d44cd3874b541c057e900b as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/ddf6b87ea3d44cd3874b541c057e900b 2024-11-07T09:36:45,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,959 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/ddf6b87ea3d44cd3874b541c057e900b, entries=150, sequenceid=319, filesize=12.0 K 2024-11-07T09:36:45,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/9b24b643d93a4b30b986c7e444ceb70a as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/9b24b643d93a4b30b986c7e444ceb70a 2024-11-07T09:36:45,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,966 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/9b24b643d93a4b30b986c7e444ceb70a, entries=150, sequenceid=319, filesize=12.0 K 2024-11-07T09:36:45,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,967 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=20.13 KB/20610 for 81b216f4bb32d787c444da8261b91891 in 913ms, sequenceid=319, compaction requested=true 2024-11-07T09:36:45,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81b216f4bb32d787c444da8261b91891: 2024-11-07T09:36:45,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81b216f4bb32d787c444da8261b91891:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T09:36:45,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:45,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81b216f4bb32d787c444da8261b91891:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T09:36:45,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:36:45,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,967 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:36:45,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81b216f4bb32d787c444da8261b91891:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:36:45,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T09:36:45,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,968 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:36:45,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,969 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:36:45,969 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:36:45,969 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 81b216f4bb32d787c444da8261b91891/B is initiating minor compaction (all files) 2024-11-07T09:36:45,969 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 81b216f4bb32d787c444da8261b91891/A is initiating minor compaction (all files) 2024-11-07T09:36:45,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,969 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81b216f4bb32d787c444da8261b91891/A in TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:45,969 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81b216f4bb32d787c444da8261b91891/B in TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:45,969 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/a46b734b757b479099e69792d028cfd6, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/b56c79bca4324917a98823c47afb4c7e, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/ddf6b87ea3d44cd3874b541c057e900b] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp, totalSize=34.3 K 2024-11-07T09:36:45,969 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/3a6716d59a4946f2976e15f1e238a9cd, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/3144bb6601124f0192cfb04c629890b5, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/f6b59c7ad90a4f2281529ad52ebe4aca] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp, totalSize=92.2 K 2024-11-07T09:36:45,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,969 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:45,969 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. files: [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/3a6716d59a4946f2976e15f1e238a9cd, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/3144bb6601124f0192cfb04c629890b5, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/f6b59c7ad90a4f2281529ad52ebe4aca] 2024-11-07T09:36:45,970 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting a46b734b757b479099e69792d028cfd6, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1730972202520 2024-11-07T09:36:45,970 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a6716d59a4946f2976e15f1e238a9cd, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1730972202520 2024-11-07T09:36:45,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,970 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting b56c79bca4324917a98823c47afb4c7e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1730972202643 2024-11-07T09:36:45,971 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3144bb6601124f0192cfb04c629890b5, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1730972202643 2024-11-07T09:36:45,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,971 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6b59c7ad90a4f2281529ad52ebe4aca, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1730972203902 2024-11-07T09:36:45,971 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting ddf6b87ea3d44cd3874b541c057e900b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1730972203917 2024-11-07T09:36:45,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-07T09:36:45,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:45,983 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing 81b216f4bb32d787c444da8261b91891 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-07T09:36:45,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,997 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=81b216f4bb32d787c444da8261b91891] 2024-11-07T09:36:45,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:45,999 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81b216f4bb32d787c444da8261b91891#B#compaction#186 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:46,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,000 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/e03afeea4f1b4447b2808f39fcf3727e is 50, key is test_row_0/B:col10/1730972205054/Put/seqid=0 2024-11-07T09:36:46,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,004 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411076a5d9f6222bc40e3b8e42bfb8287388e_81b216f4bb32d787c444da8261b91891 store=[table=TestAcidGuarantees family=A region=81b216f4bb32d787c444da8261b91891] 2024-11-07T09:36:46,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107668ef98ebbc64606ada2714aa545130b_81b216f4bb32d787c444da8261b91891 is 50, key is test_row_0/A:col10/1730972205055/Put/seqid=0 2024-11-07T09:36:46,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,006 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411076a5d9f6222bc40e3b8e42bfb8287388e_81b216f4bb32d787c444da8261b91891, store=[table=TestAcidGuarantees family=A region=81b216f4bb32d787c444da8261b91891] 2024-11-07T09:36:46,006 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411076a5d9f6222bc40e3b8e42bfb8287388e_81b216f4bb32d787c444da8261b91891 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=81b216f4bb32d787c444da8261b91891] 2024-11-07T09:36:46,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742049_1225 (size=4469) 2024-11-07T09:36:46,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,043 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81b216f4bb32d787c444da8261b91891#A#compaction#187 average throughput is 0.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:46,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,044 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/614b3593b9b5423a9058ec95fb19f1a9 is 175, key is test_row_0/A:col10/1730972205054/Put/seqid=0 2024-11-07T09:36:46,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742047_1223 (size=13017) 2024-11-07T09:36:46,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742048_1224 (size=9914) 2024-11-07T09:36:46,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742050_1226 (size=31971) 2024-11-07T09:36:46,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. as already flushing 2024-11-07T09:36:46,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:46,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:46,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972266169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:46,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:46,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:46,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:46,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:46,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972266171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:46,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:46,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:46,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:46,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:46,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972266203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:46,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:46,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:46,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:46,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972266203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:46,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:46,209 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:46,209 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:46,209 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:46,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:46,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972266283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:46,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:46,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:46,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:46,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:46,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972266283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:46,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:46,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:46,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:46,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-07T09:36:46,455 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/e03afeea4f1b4447b2808f39fcf3727e as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/e03afeea4f1b4447b2808f39fcf3727e 2024-11-07T09:36:46,462 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81b216f4bb32d787c444da8261b91891/B of 81b216f4bb32d787c444da8261b91891 into e03afeea4f1b4447b2808f39fcf3727e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:46,462 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81b216f4bb32d787c444da8261b91891: 2024-11-07T09:36:46,462 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., storeName=81b216f4bb32d787c444da8261b91891/B, priority=13, startTime=1730972205967; duration=0sec 2024-11-07T09:36:46,462 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:36:46,462 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81b216f4bb32d787c444da8261b91891:B 2024-11-07T09:36:46,462 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:36:46,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:46,463 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:36:46,463 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 81b216f4bb32d787c444da8261b91891/C is initiating minor compaction (all files) 2024-11-07T09:36:46,464 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81b216f4bb32d787c444da8261b91891/C in TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:46,464 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/cf2bd8fc1343428494f209991176bc61, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/98b5a61ea61b483e88d068e4f148e0e3, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/9b24b643d93a4b30b986c7e444ceb70a] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp, totalSize=34.3 K 2024-11-07T09:36:46,465 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting cf2bd8fc1343428494f209991176bc61, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1730972202520 2024-11-07T09:36:46,466 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 98b5a61ea61b483e88d068e4f148e0e3, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1730972202643 2024-11-07T09:36:46,466 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b24b643d93a4b30b986c7e444ceb70a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1730972203917 2024-11-07T09:36:46,467 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107668ef98ebbc64606ada2714aa545130b_81b216f4bb32d787c444da8261b91891 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107668ef98ebbc64606ada2714aa545130b_81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:46,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/0029131742fe444faae6c4df7182f510, store: [table=TestAcidGuarantees family=A region=81b216f4bb32d787c444da8261b91891] 2024-11-07T09:36:46,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/0029131742fe444faae6c4df7182f510 is 175, key is test_row_0/A:col10/1730972205055/Put/seqid=0 2024-11-07T09:36:46,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742051_1227 (size=22561) 2024-11-07T09:36:46,475 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=325, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/0029131742fe444faae6c4df7182f510 2024-11-07T09:36:46,488 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81b216f4bb32d787c444da8261b91891#C#compaction#189 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:46,488 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/7223d81b026249f0a95bb35f7e078c9a is 50, key is test_row_0/C:col10/1730972205054/Put/seqid=0 2024-11-07T09:36:46,493 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/614b3593b9b5423a9058ec95fb19f1a9 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/614b3593b9b5423a9058ec95fb19f1a9 2024-11-07T09:36:46,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:46,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972266493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:46,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:46,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972266493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:46,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:46,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:46,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:46,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:46,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:46,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:46,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/d5b5cf47de854e09ae8aecd980df45eb is 50, key is test_row_0/B:col10/1730972205055/Put/seqid=0 2024-11-07T09:36:46,503 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81b216f4bb32d787c444da8261b91891/A of 81b216f4bb32d787c444da8261b91891 into 614b3593b9b5423a9058ec95fb19f1a9(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:46,503 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81b216f4bb32d787c444da8261b91891: 2024-11-07T09:36:46,503 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., storeName=81b216f4bb32d787c444da8261b91891/A, priority=13, startTime=1730972205967; duration=0sec 2024-11-07T09:36:46,503 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:46,503 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81b216f4bb32d787c444da8261b91891:A 2024-11-07T09:36:46,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742052_1228 (size=9857) 2024-11-07T09:36:46,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742053_1229 (size=13017) 2024-11-07T09:36:46,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:46,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972266803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:46,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:46,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972266803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:46,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:46,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:46,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:46,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:46,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:46,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:46,919 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/d5b5cf47de854e09ae8aecd980df45eb 2024-11-07T09:36:46,927 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/7223d81b026249f0a95bb35f7e078c9a as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/7223d81b026249f0a95bb35f7e078c9a 2024-11-07T09:36:46,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/c560e4b8f1a94731acd5b192ddf8382b is 50, key is test_row_0/C:col10/1730972205055/Put/seqid=0 2024-11-07T09:36:46,936 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81b216f4bb32d787c444da8261b91891/C of 81b216f4bb32d787c444da8261b91891 into 7223d81b026249f0a95bb35f7e078c9a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:46,936 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81b216f4bb32d787c444da8261b91891: 2024-11-07T09:36:46,936 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., storeName=81b216f4bb32d787c444da8261b91891/C, priority=13, startTime=1730972205967; duration=0sec 2024-11-07T09:36:46,936 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:46,936 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81b216f4bb32d787c444da8261b91891:C 2024-11-07T09:36:46,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742054_1230 (size=9857) 2024-11-07T09:36:46,949 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/c560e4b8f1a94731acd5b192ddf8382b 2024-11-07T09:36:46,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/0029131742fe444faae6c4df7182f510 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/0029131742fe444faae6c4df7182f510 2024-11-07T09:36:46,958 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/0029131742fe444faae6c4df7182f510, entries=100, sequenceid=325, filesize=22.0 K 2024-11-07T09:36:46,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/d5b5cf47de854e09ae8aecd980df45eb as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/d5b5cf47de854e09ae8aecd980df45eb 2024-11-07T09:36:46,964 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/d5b5cf47de854e09ae8aecd980df45eb, entries=100, sequenceid=325, filesize=9.6 K 2024-11-07T09:36:46,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/c560e4b8f1a94731acd5b192ddf8382b as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/c560e4b8f1a94731acd5b192ddf8382b 2024-11-07T09:36:46,970 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/c560e4b8f1a94731acd5b192ddf8382b, entries=100, sequenceid=325, filesize=9.6 K 2024-11-07T09:36:46,971 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=181.14 KB/185490 for 81b216f4bb32d787c444da8261b91891 in 988ms, sequenceid=325, compaction requested=false 2024-11-07T09:36:46,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 81b216f4bb32d787c444da8261b91891: 2024-11-07T09:36:46,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:46,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-07T09:36:46,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-07T09:36:46,980 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-07T09:36:46,980 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7610 sec 2024-11-07T09:36:46,982 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 1.7660 sec 2024-11-07T09:36:47,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:47,214 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:36:47,215 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81b216f4bb32d787c444da8261b91891 3/3 column families, dataSize=187.85 KB heapSize=492.94 KB 2024-11-07T09:36:47,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107f829067f54ae474bbef7f068a0962a9a_81b216f4bb32d787c444da8261b91891 is 50, key is test_row_0/A:col10/1730972207213/Put/seqid=0 2024-11-07T09:36:47,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:47,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972267223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:47,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:47,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972267223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:47,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742055_1231 (size=14994) 2024-11-07T09:36:47,227 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 9f1eb259a29b,42895,1730972153970 for 'test_row_1' in 81b216f4bb32d787c444da8261b91891 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T09:36:47,228 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:47,228 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:47,228 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:47,228 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:47,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:47,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:47,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:47,231 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107f829067f54ae474bbef7f068a0962a9a_81b216f4bb32d787c444da8261b91891 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f829067f54ae474bbef7f068a0962a9a_81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:47,233 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/11fa9417932c4b10936ff129651ea4dc, store: [table=TestAcidGuarantees family=A region=81b216f4bb32d787c444da8261b91891] 2024-11-07T09:36:47,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/11fa9417932c4b10936ff129651ea4dc is 175, key is test_row_0/A:col10/1730972207213/Put/seqid=0 2024-11-07T09:36:47,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742056_1232 (size=39949) 2024-11-07T09:36:47,238 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=359, memsize=62.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/11fa9417932c4b10936ff129651ea4dc 2024-11-07T09:36:47,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/9b1838f1d8cf457f8572e8c0129ce706 is 50, key is test_row_0/B:col10/1730972207213/Put/seqid=0 2024-11-07T09:36:47,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742057_1233 (size=12301) 2024-11-07T09:36:47,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:47,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972267313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:47,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:47,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:47,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972267313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:47,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:47,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:47,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:47,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:47,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:47,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:47,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972267332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:47,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:47,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:47,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:47,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-07T09:36:47,363 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:36:47,363 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T09:36:47,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-07T09:36:47,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-07T09:36:47,365 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T09:36:47,366 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T09:36:47,366 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T09:36:47,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-07T09:36:47,518 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-07T09:36:47,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:47,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. as already flushing 2024-11-07T09:36:47,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:47,519 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:47,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:47,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:47,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:47,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972267543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:47,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:47,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:47,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:47,673 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/9b1838f1d8cf457f8572e8c0129ce706 2024-11-07T09:36:47,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-07T09:36:47,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:47,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. as already flushing 2024-11-07T09:36:47,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:47,674 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:47,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:47,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:47,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-07T09:36:47,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/61fb62057b864724acc18f17f2333415 is 50, key is test_row_0/C:col10/1730972207213/Put/seqid=0 2024-11-07T09:36:47,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742058_1234 (size=12301) 2024-11-07T09:36:47,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-07T09:36:47,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:47,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. as already flushing 2024-11-07T09:36:47,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:47,827 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:47,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:47,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:47,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:47,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972267853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:47,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:47,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:47,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:47,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-07T09:36:47,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:47,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. as already flushing 2024-11-07T09:36:47,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:47,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:47,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:47,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:36:47,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-07T09:36:48,103 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/61fb62057b864724acc18f17f2333415 2024-11-07T09:36:48,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/11fa9417932c4b10936ff129651ea4dc as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/11fa9417932c4b10936ff129651ea4dc 2024-11-07T09:36:48,113 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/11fa9417932c4b10936ff129651ea4dc, entries=200, sequenceid=359, filesize=39.0 K 2024-11-07T09:36:48,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/9b1838f1d8cf457f8572e8c0129ce706 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/9b1838f1d8cf457f8572e8c0129ce706 2024-11-07T09:36:48,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,121 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/9b1838f1d8cf457f8572e8c0129ce706, entries=150, sequenceid=359, filesize=12.0 K 2024-11-07T09:36:48,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/61fb62057b864724acc18f17f2333415 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/61fb62057b864724acc18f17f2333415 2024-11-07T09:36:48,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,128 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/61fb62057b864724acc18f17f2333415, entries=150, sequenceid=359, filesize=12.0 K 2024-11-07T09:36:48,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,129 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~187.85 KB/192360, heapSize ~492.89 KB/504720, currentSize=13.42 KB/13740 for 81b216f4bb32d787c444da8261b91891 in 915ms, sequenceid=359, compaction requested=true 2024-11-07T09:36:48,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81b216f4bb32d787c444da8261b91891: 2024-11-07T09:36:48,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81b216f4bb32d787c444da8261b91891:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T09:36:48,129 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:36:48,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:48,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81b216f4bb32d787c444da8261b91891:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T09:36:48,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:48,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81b216f4bb32d787c444da8261b91891:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:36:48,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:36:48,129 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:36:48,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,130 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94481 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:36:48,130 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 81b216f4bb32d787c444da8261b91891/A is initiating minor compaction (all files) 2024-11-07T09:36:48,131 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:36:48,131 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81b216f4bb32d787c444da8261b91891/A in TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:48,131 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 81b216f4bb32d787c444da8261b91891/B is initiating minor compaction (all files) 2024-11-07T09:36:48,131 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/614b3593b9b5423a9058ec95fb19f1a9, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/0029131742fe444faae6c4df7182f510, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/11fa9417932c4b10936ff129651ea4dc] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp, totalSize=92.3 K 2024-11-07T09:36:48,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,131 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:48,131 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81b216f4bb32d787c444da8261b91891/B in TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:48,131 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. files: [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/614b3593b9b5423a9058ec95fb19f1a9, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/0029131742fe444faae6c4df7182f510, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/11fa9417932c4b10936ff129651ea4dc] 2024-11-07T09:36:48,131 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/e03afeea4f1b4447b2808f39fcf3727e, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/d5b5cf47de854e09ae8aecd980df45eb, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/9b1838f1d8cf457f8572e8c0129ce706] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp, totalSize=34.4 K 2024-11-07T09:36:48,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,131 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 614b3593b9b5423a9058ec95fb19f1a9, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1730972203917 2024-11-07T09:36:48,131 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting e03afeea4f1b4447b2808f39fcf3727e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1730972203917 2024-11-07T09:36:48,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,132 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0029131742fe444faae6c4df7182f510, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1730972205055 2024-11-07T09:36:48,132 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting d5b5cf47de854e09ae8aecd980df45eb, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1730972205055 2024-11-07T09:36:48,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,132 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11fa9417932c4b10936ff129651ea4dc, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1730972206155 2024-11-07T09:36:48,132 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b1838f1d8cf457f8572e8c0129ce706, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1730972206168 2024-11-07T09:36:48,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-07T09:36:48,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:48,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,134 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 81b216f4bb32d787c444da8261b91891 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-07T09:36:48,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,149 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=81b216f4bb32d787c444da8261b91891] 2024-11-07T09:36:48,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110743a1c59907954c989974af6fd3364b39_81b216f4bb32d787c444da8261b91891 is 50, key is test_row_0/A:col10/1730972207221/Put/seqid=0 2024-11-07T09:36:48,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,151 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81b216f4bb32d787c444da8261b91891#B#compaction#197 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:48,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,152 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/64be9c2090e44425b7a37db26d73cf79 is 50, key is test_row_0/B:col10/1730972207213/Put/seqid=0 2024-11-07T09:36:48,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,162 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107fd033123a91048fd8d6f93856e07d4ee_81b216f4bb32d787c444da8261b91891 store=[table=TestAcidGuarantees family=A region=81b216f4bb32d787c444da8261b91891] 2024-11-07T09:36:48,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,164 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107fd033123a91048fd8d6f93856e07d4ee_81b216f4bb32d787c444da8261b91891, store=[table=TestAcidGuarantees family=A region=81b216f4bb32d787c444da8261b91891] 2024-11-07T09:36:48,164 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107fd033123a91048fd8d6f93856e07d4ee_81b216f4bb32d787c444da8261b91891 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=81b216f4bb32d787c444da8261b91891] 2024-11-07T09:36:48,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742061_1237 (size=4469) 2024-11-07T09:36:48,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742059_1235 (size=9914) 2024-11-07T09:36:48,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742060_1236 (size=13119) 2024-11-07T09:36:48,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:48,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. as already flushing 2024-11-07T09:36:48,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:48,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972268434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:48,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:48,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:48,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:48,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:48,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972268435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:48,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:48,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:48,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:48,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:48,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972268440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:48,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:48,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:48,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:48,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-07T09:36:48,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:48,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972268543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:48,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:48,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:48,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:48,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:48,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972268552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:48,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:48,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:48,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:48,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972268552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:48,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:48,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:48,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:48,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:48,572 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81b216f4bb32d787c444da8261b91891#A#compaction#196 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:48,573 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/bfe5e744fb65416ab8a426ab37e16c68 is 175, key is test_row_0/A:col10/1730972207213/Put/seqid=0 2024-11-07T09:36:48,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:48,580 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110743a1c59907954c989974af6fd3364b39_81b216f4bb32d787c444da8261b91891 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110743a1c59907954c989974af6fd3364b39_81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:48,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742062_1238 (size=32073) 2024-11-07T09:36:48,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/405591038c6f4220b8123312447d80b7, store: [table=TestAcidGuarantees family=A region=81b216f4bb32d787c444da8261b91891] 2024-11-07T09:36:48,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/405591038c6f4220b8123312447d80b7 is 175, key is test_row_0/A:col10/1730972207221/Put/seqid=0 2024-11-07T09:36:48,585 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/64be9c2090e44425b7a37db26d73cf79 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/64be9c2090e44425b7a37db26d73cf79 2024-11-07T09:36:48,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742063_1239 (size=22561) 2024-11-07T09:36:48,592 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81b216f4bb32d787c444da8261b91891/B of 81b216f4bb32d787c444da8261b91891 into 64be9c2090e44425b7a37db26d73cf79(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:48,592 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81b216f4bb32d787c444da8261b91891: 2024-11-07T09:36:48,592 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., storeName=81b216f4bb32d787c444da8261b91891/B, priority=13, startTime=1730972208129; duration=0sec 2024-11-07T09:36:48,592 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:36:48,592 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81b216f4bb32d787c444da8261b91891:B 2024-11-07T09:36:48,592 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:36:48,593 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:36:48,593 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 81b216f4bb32d787c444da8261b91891/C is initiating minor compaction (all files) 2024-11-07T09:36:48,594 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81b216f4bb32d787c444da8261b91891/C in TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:48,594 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/7223d81b026249f0a95bb35f7e078c9a, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/c560e4b8f1a94731acd5b192ddf8382b, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/61fb62057b864724acc18f17f2333415] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp, totalSize=34.4 K 2024-11-07T09:36:48,594 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 7223d81b026249f0a95bb35f7e078c9a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1730972203917 2024-11-07T09:36:48,594 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting c560e4b8f1a94731acd5b192ddf8382b, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1730972205055 2024-11-07T09:36:48,595 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 61fb62057b864724acc18f17f2333415, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1730972206168 2024-11-07T09:36:48,603 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81b216f4bb32d787c444da8261b91891#C#compaction#198 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:48,603 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/0922d9b6cc71442a993903011433504e is 50, key is test_row_0/C:col10/1730972207213/Put/seqid=0 2024-11-07T09:36:48,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742064_1240 (size=13119) 2024-11-07T09:36:48,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:48,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1730972268753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:48,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:48,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972268753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:48,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:48,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:48,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:48,758 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 9f1eb259a29b,42895,1730972153970 for 'test_row_2' in 81b216f4bb32d787c444da8261b91891 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T09:36:48,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:48,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:48,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:48,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:48,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972268762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:48,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:48,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:48,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:48,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:48,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972268762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:48,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:48,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:48,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:48,993 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=364, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/405591038c6f4220b8123312447d80b7 2024-11-07T09:36:48,996 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/bfe5e744fb65416ab8a426ab37e16c68 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/bfe5e744fb65416ab8a426ab37e16c68 2024-11-07T09:36:49,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/c6f29b4465564b0e9a8c27387b8982bd is 50, key is test_row_0/B:col10/1730972207221/Put/seqid=0 2024-11-07T09:36:49,004 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81b216f4bb32d787c444da8261b91891/A of 81b216f4bb32d787c444da8261b91891 into bfe5e744fb65416ab8a426ab37e16c68(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:49,004 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81b216f4bb32d787c444da8261b91891: 2024-11-07T09:36:49,004 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., storeName=81b216f4bb32d787c444da8261b91891/A, priority=13, startTime=1730972208129; duration=0sec 2024-11-07T09:36:49,004 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:49,004 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81b216f4bb32d787c444da8261b91891:A 2024-11-07T09:36:49,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742065_1241 (size=9857) 2024-11-07T09:36:49,006 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/c6f29b4465564b0e9a8c27387b8982bd 2024-11-07T09:36:49,012 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/0922d9b6cc71442a993903011433504e as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/0922d9b6cc71442a993903011433504e 2024-11-07T09:36:49,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/6f989abcf8624df99a83942dd4fa43fc is 50, key is test_row_0/C:col10/1730972207221/Put/seqid=0 2024-11-07T09:36:49,019 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81b216f4bb32d787c444da8261b91891/C of 81b216f4bb32d787c444da8261b91891 into 0922d9b6cc71442a993903011433504e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:49,019 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81b216f4bb32d787c444da8261b91891: 2024-11-07T09:36:49,019 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., storeName=81b216f4bb32d787c444da8261b91891/C, priority=13, startTime=1730972208129; duration=0sec 2024-11-07T09:36:49,019 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:49,019 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81b216f4bb32d787c444da8261b91891:C 2024-11-07T09:36:49,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742066_1242 (size=9857) 2024-11-07T09:36:49,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:49,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37602 deadline: 1730972269063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:49,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:49,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:49,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:49,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:49,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37640 deadline: 1730972269073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:49,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:49,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:49,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:49,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:49,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37622 deadline: 1730972269073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:49,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:49,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:49,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:49,150 INFO [Thread-695 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-695. 2024-11-07T09:36:49,150 DEBUG [Thread-695 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:36:49,150 DEBUG [Thread-695 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:49,150 DEBUG [Thread-695 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:49,150 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:36:49,151 INFO [Thread-693 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-693. 2024-11-07T09:36:49,151 DEBUG [Thread-693 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:36:49,151 DEBUG [Thread-693 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:49,151 DEBUG [Thread-693 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:49,151 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:36:49,155 INFO [Thread-699 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-699. 2024-11-07T09:36:49,155 DEBUG [Thread-699 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:36:49,155 DEBUG [Thread-699 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:49,155 DEBUG [Thread-699 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:49,155 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:36:49,157 INFO [Thread-697 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-697. 2024-11-07T09:36:49,157 DEBUG [Thread-697 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:36:49,157 DEBUG [Thread-697 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:49,158 DEBUG [Thread-697 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:49,158 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:36:49,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:36:49,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1730972269253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:49,254 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 9f1eb259a29b,42895,1730972153970 for 'test_row_1' in 81b216f4bb32d787c444da8261b91891 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T09:36:49,254 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:49,254 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:36:49,254 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:36:49,422 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/6f989abcf8624df99a83942dd4fa43fc 2024-11-07T09:36:49,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/405591038c6f4220b8123312447d80b7 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/405591038c6f4220b8123312447d80b7 2024-11-07T09:36:49,430 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/405591038c6f4220b8123312447d80b7, entries=100, sequenceid=364, filesize=22.0 K 2024-11-07T09:36:49,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/c6f29b4465564b0e9a8c27387b8982bd as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/c6f29b4465564b0e9a8c27387b8982bd 2024-11-07T09:36:49,434 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/c6f29b4465564b0e9a8c27387b8982bd, entries=100, sequenceid=364, filesize=9.6 K 2024-11-07T09:36:49,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/6f989abcf8624df99a83942dd4fa43fc as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/6f989abcf8624df99a83942dd4fa43fc 2024-11-07T09:36:49,438 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/6f989abcf8624df99a83942dd4fa43fc, entries=100, sequenceid=364, filesize=9.6 K 2024-11-07T09:36:49,439 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=187.85 KB/192360 for 81b216f4bb32d787c444da8261b91891 in 1305ms, sequenceid=364, compaction requested=false 2024-11-07T09:36:49,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 81b216f4bb32d787c444da8261b91891: 2024-11-07T09:36:49,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:49,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-07T09:36:49,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-07T09:36:49,442 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-07T09:36:49,442 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0750 sec 2024-11-07T09:36:49,444 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 2.0790 sec 2024-11-07T09:36:49,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-07T09:36:49,513 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:36:49,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:49,574 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:36:49,574 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81b216f4bb32d787c444da8261b91891 3/3 column families, dataSize=194.56 KB heapSize=510.52 KB 2024-11-07T09:36:49,574 INFO [Thread-686 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-686. 2024-11-07T09:36:49,574 DEBUG [Thread-686 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:36:49,574 DEBUG [Thread-686 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:49,574 DEBUG [Thread-686 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:49,574 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:36:49,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107e2cc54ff980f43d9899c3db2db091ad5_81b216f4bb32d787c444da8261b91891 is 50, key is test_row_0/A:col10/1730972208439/Put/seqid=0 2024-11-07T09:36:49,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742067_1243 (size=12454) 2024-11-07T09:36:49,584 INFO [Thread-690 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-690. 2024-11-07T09:36:49,584 DEBUG [Thread-690 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:36:49,584 DEBUG [Thread-690 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:49,584 INFO [Thread-688 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-688. 2024-11-07T09:36:49,584 DEBUG [Thread-690 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:49,584 DEBUG [Thread-688 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:36:49,584 DEBUG [Thread-688 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:49,584 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:36:49,584 DEBUG [Thread-688 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:49,584 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:36:49,984 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:49,988 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107e2cc54ff980f43d9899c3db2db091ad5_81b216f4bb32d787c444da8261b91891 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107e2cc54ff980f43d9899c3db2db091ad5_81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:49,989 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/0cdfe8970a344a6aafb7dee200820400, store: [table=TestAcidGuarantees family=A region=81b216f4bb32d787c444da8261b91891] 2024-11-07T09:36:49,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/0cdfe8970a344a6aafb7dee200820400 is 175, key is test_row_0/A:col10/1730972208439/Put/seqid=0 2024-11-07T09:36:49,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742068_1244 (size=31255) 2024-11-07T09:36:50,394 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=399, memsize=64.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/0cdfe8970a344a6aafb7dee200820400 2024-11-07T09:36:50,401 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/ed678bee40b642b393f5292fd41bd825 is 50, key is test_row_0/B:col10/1730972208439/Put/seqid=0 2024-11-07T09:36:50,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742069_1245 (size=12301) 2024-11-07T09:36:50,805 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/ed678bee40b642b393f5292fd41bd825 2024-11-07T09:36:50,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/f14ff173a7d84c0497924fa9425bb4a6 is 50, key is test_row_0/C:col10/1730972208439/Put/seqid=0 2024-11-07T09:36:50,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742070_1246 (size=12301) 2024-11-07T09:36:51,223 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/f14ff173a7d84c0497924fa9425bb4a6 2024-11-07T09:36:51,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/0cdfe8970a344a6aafb7dee200820400 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/0cdfe8970a344a6aafb7dee200820400 2024-11-07T09:36:51,232 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/0cdfe8970a344a6aafb7dee200820400, entries=150, sequenceid=399, filesize=30.5 K 2024-11-07T09:36:51,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/ed678bee40b642b393f5292fd41bd825 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/ed678bee40b642b393f5292fd41bd825 2024-11-07T09:36:51,237 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/ed678bee40b642b393f5292fd41bd825, entries=150, sequenceid=399, filesize=12.0 K 2024-11-07T09:36:51,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/f14ff173a7d84c0497924fa9425bb4a6 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/f14ff173a7d84c0497924fa9425bb4a6 2024-11-07T09:36:51,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/f14ff173a7d84c0497924fa9425bb4a6, entries=150, sequenceid=399, filesize=12.0 K 2024-11-07T09:36:51,243 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~194.56 KB/199230, heapSize ~510.47 KB/522720, currentSize=13.42 KB/13740 for 81b216f4bb32d787c444da8261b91891 in 1669ms, sequenceid=399, compaction requested=true 2024-11-07T09:36:51,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81b216f4bb32d787c444da8261b91891: 2024-11-07T09:36:51,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81b216f4bb32d787c444da8261b91891:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T09:36:51,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:51,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81b216f4bb32d787c444da8261b91891:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T09:36:51,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:51,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81b216f4bb32d787c444da8261b91891:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:36:51,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:36:51,243 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:36:51,243 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:36:51,244 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 85889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:36:51,244 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:36:51,244 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 81b216f4bb32d787c444da8261b91891/A is initiating minor compaction (all files) 2024-11-07T09:36:51,244 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 81b216f4bb32d787c444da8261b91891/B is initiating minor compaction (all files) 2024-11-07T09:36:51,244 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81b216f4bb32d787c444da8261b91891/A in TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:51,244 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81b216f4bb32d787c444da8261b91891/B in TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:51,244 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/bfe5e744fb65416ab8a426ab37e16c68, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/405591038c6f4220b8123312447d80b7, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/0cdfe8970a344a6aafb7dee200820400] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp, totalSize=83.9 K 2024-11-07T09:36:51,244 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:51,244 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/64be9c2090e44425b7a37db26d73cf79, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/c6f29b4465564b0e9a8c27387b8982bd, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/ed678bee40b642b393f5292fd41bd825] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp, totalSize=34.5 K 2024-11-07T09:36:51,244 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. files: [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/bfe5e744fb65416ab8a426ab37e16c68, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/405591038c6f4220b8123312447d80b7, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/0cdfe8970a344a6aafb7dee200820400] 2024-11-07T09:36:51,245 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfe5e744fb65416ab8a426ab37e16c68, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1730972206168 2024-11-07T09:36:51,245 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 64be9c2090e44425b7a37db26d73cf79, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1730972206168 2024-11-07T09:36:51,245 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 405591038c6f4220b8123312447d80b7, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1730972207218 2024-11-07T09:36:51,245 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting c6f29b4465564b0e9a8c27387b8982bd, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1730972207218 2024-11-07T09:36:51,245 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0cdfe8970a344a6aafb7dee200820400, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1730972208431 2024-11-07T09:36:51,246 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting ed678bee40b642b393f5292fd41bd825, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1730972208431 2024-11-07T09:36:51,253 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=81b216f4bb32d787c444da8261b91891] 2024-11-07T09:36:51,254 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81b216f4bb32d787c444da8261b91891#B#compaction#204 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:51,255 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/638ccf7f058d4a5ab77ae255955ed1c9 is 50, key is test_row_0/B:col10/1730972208439/Put/seqid=0 2024-11-07T09:36:51,255 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107ba2f5b55352646eca6c5dcba47b2c14a_81b216f4bb32d787c444da8261b91891 store=[table=TestAcidGuarantees family=A region=81b216f4bb32d787c444da8261b91891] 2024-11-07T09:36:51,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742071_1247 (size=13221) 2024-11-07T09:36:51,275 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/638ccf7f058d4a5ab77ae255955ed1c9 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/638ccf7f058d4a5ab77ae255955ed1c9 2024-11-07T09:36:51,276 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107ba2f5b55352646eca6c5dcba47b2c14a_81b216f4bb32d787c444da8261b91891, store=[table=TestAcidGuarantees family=A region=81b216f4bb32d787c444da8261b91891] 2024-11-07T09:36:51,276 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107ba2f5b55352646eca6c5dcba47b2c14a_81b216f4bb32d787c444da8261b91891 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=81b216f4bb32d787c444da8261b91891] 2024-11-07T09:36:51,282 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81b216f4bb32d787c444da8261b91891/B of 81b216f4bb32d787c444da8261b91891 into 638ccf7f058d4a5ab77ae255955ed1c9(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:51,282 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81b216f4bb32d787c444da8261b91891: 2024-11-07T09:36:51,282 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., storeName=81b216f4bb32d787c444da8261b91891/B, priority=13, startTime=1730972211243; duration=0sec 2024-11-07T09:36:51,282 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:36:51,282 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81b216f4bb32d787c444da8261b91891:B 2024-11-07T09:36:51,282 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:36:51,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742072_1248 (size=4469) 2024-11-07T09:36:51,284 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:36:51,284 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 81b216f4bb32d787c444da8261b91891/C is initiating minor compaction (all files) 2024-11-07T09:36:51,284 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81b216f4bb32d787c444da8261b91891/C in TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:51,284 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/0922d9b6cc71442a993903011433504e, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/6f989abcf8624df99a83942dd4fa43fc, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/f14ff173a7d84c0497924fa9425bb4a6] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp, totalSize=34.5 K 2024-11-07T09:36:51,285 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81b216f4bb32d787c444da8261b91891#A#compaction#205 average throughput is 0.76 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:51,285 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 0922d9b6cc71442a993903011433504e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1730972206168 2024-11-07T09:36:51,286 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f989abcf8624df99a83942dd4fa43fc, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1730972207218 2024-11-07T09:36:51,286 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/aa0e0068d5544e24a8955f72cb7944bd is 175, key is test_row_0/A:col10/1730972208439/Put/seqid=0 2024-11-07T09:36:51,287 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting f14ff173a7d84c0497924fa9425bb4a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1730972208431 2024-11-07T09:36:51,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742073_1249 (size=32175) 2024-11-07T09:36:51,302 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81b216f4bb32d787c444da8261b91891#C#compaction#206 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:36:51,302 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/9cba5593f4de4f32905ab9a44b2ef09d is 50, key is test_row_0/C:col10/1730972208439/Put/seqid=0 2024-11-07T09:36:51,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742074_1250 (size=13221) 2024-11-07T09:36:51,699 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/aa0e0068d5544e24a8955f72cb7944bd as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/aa0e0068d5544e24a8955f72cb7944bd 2024-11-07T09:36:51,705 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81b216f4bb32d787c444da8261b91891/A of 81b216f4bb32d787c444da8261b91891 into aa0e0068d5544e24a8955f72cb7944bd(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:51,705 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81b216f4bb32d787c444da8261b91891: 2024-11-07T09:36:51,705 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., storeName=81b216f4bb32d787c444da8261b91891/A, priority=13, startTime=1730972211243; duration=0sec 2024-11-07T09:36:51,705 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:51,705 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81b216f4bb32d787c444da8261b91891:A 2024-11-07T09:36:51,715 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/9cba5593f4de4f32905ab9a44b2ef09d as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/9cba5593f4de4f32905ab9a44b2ef09d 2024-11-07T09:36:51,721 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81b216f4bb32d787c444da8261b91891/C of 81b216f4bb32d787c444da8261b91891 into 9cba5593f4de4f32905ab9a44b2ef09d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:36:51,721 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81b216f4bb32d787c444da8261b91891: 2024-11-07T09:36:51,721 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., storeName=81b216f4bb32d787c444da8261b91891/C, priority=13, startTime=1730972211243; duration=0sec 2024-11-07T09:36:51,721 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:36:51,721 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81b216f4bb32d787c444da8261b91891:C 2024-11-07T09:36:52,442 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T09:36:53,275 INFO [Thread-684 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-684. 2024-11-07T09:36:53,275 DEBUG [Thread-684 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:36:53,275 DEBUG [Thread-684 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:53,275 DEBUG [Thread-684 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:53,275 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:36:58,784 INFO [Thread-682 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-682. 2024-11-07T09:36:58,784 DEBUG [Thread-682 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:36:58,784 DEBUG [Thread-682 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:58,784 DEBUG [Thread-682 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:58,784 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:36:58,785 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-07T09:36:58,785 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-11-07T09:36:58,785 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-07T09:36:58,785 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-11-07T09:36:58,785 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-07T09:36:58,785 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 92 2024-11-07T09:36:58,785 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-07T09:36:58,785 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3386 2024-11-07T09:36:58,785 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3305 2024-11-07T09:36:58,785 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-07T09:36:58,785 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1465 2024-11-07T09:36:58,785 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4392 rows 2024-11-07T09:36:58,785 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1456 2024-11-07T09:36:58,785 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4367 rows 2024-11-07T09:36:58,785 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T09:36:58,785 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T09:36:58,785 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:58,785 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:36:58,785 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:36:58,786 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-07T09:36:58,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-07T09:36:58,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-07T09:36:58,789 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730972218789"}]},"ts":"1730972218789"} 2024-11-07T09:36:58,790 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-07T09:36:58,790 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-07T09:36:58,791 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T09:36:58,792 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=81b216f4bb32d787c444da8261b91891, UNASSIGN}] 2024-11-07T09:36:58,793 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=81b216f4bb32d787c444da8261b91891, UNASSIGN 2024-11-07T09:36:58,794 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=81b216f4bb32d787c444da8261b91891, regionState=CLOSING, regionLocation=9f1eb259a29b,42895,1730972153970 2024-11-07T09:36:58,795 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=81b216f4bb32d787c444da8261b91891, UNASSIGN because future has completed 2024-11-07T09:36:58,795 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T09:36:58,795 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; CloseRegionProcedure 81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970}] 2024-11-07T09:36:58,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-07T09:36:58,947 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(122): Close 81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:58,947 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T09:36:58,947 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1722): Closing 81b216f4bb32d787c444da8261b91891, disabling compactions & flushes 2024-11-07T09:36:58,947 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:58,947 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:58,947 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. after waiting 0 ms 2024-11-07T09:36:58,947 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:36:58,947 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(2902): Flushing 81b216f4bb32d787c444da8261b91891 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-07T09:36:58,954 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110749d1d1714cb842bd92ef6cd10dc33f2e_81b216f4bb32d787c444da8261b91891 is 50, key is test_row_0/A:col10/1730972209583/Put/seqid=0 2024-11-07T09:36:58,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742075_1251 (size=12454) 2024-11-07T09:36:59,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-07T09:36:59,360 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:36:59,364 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110749d1d1714cb842bd92ef6cd10dc33f2e_81b216f4bb32d787c444da8261b91891 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110749d1d1714cb842bd92ef6cd10dc33f2e_81b216f4bb32d787c444da8261b91891 2024-11-07T09:36:59,365 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/23579488d92346fe88edf9a0cf0fdf48, store: [table=TestAcidGuarantees family=A region=81b216f4bb32d787c444da8261b91891] 2024-11-07T09:36:59,366 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/23579488d92346fe88edf9a0cf0fdf48 is 175, key is test_row_0/A:col10/1730972209583/Put/seqid=0 2024-11-07T09:36:59,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742076_1252 (size=31255) 2024-11-07T09:36:59,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-07T09:36:59,770 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=409, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/23579488d92346fe88edf9a0cf0fdf48 2024-11-07T09:36:59,776 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/fd1f7ddf65654ac9965fcc5d5d2c1a07 is 50, key is test_row_0/B:col10/1730972209583/Put/seqid=0 2024-11-07T09:36:59,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742077_1253 (size=12301) 2024-11-07T09:36:59,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-07T09:37:00,181 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/fd1f7ddf65654ac9965fcc5d5d2c1a07 2024-11-07T09:37:00,187 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/f00fb488689a42f4942264fec4734cc1 is 50, key is test_row_0/C:col10/1730972209583/Put/seqid=0 2024-11-07T09:37:00,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742078_1254 (size=12301) 2024-11-07T09:37:00,592 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/f00fb488689a42f4942264fec4734cc1 2024-11-07T09:37:00,596 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/A/23579488d92346fe88edf9a0cf0fdf48 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/23579488d92346fe88edf9a0cf0fdf48 2024-11-07T09:37:00,600 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/23579488d92346fe88edf9a0cf0fdf48, entries=150, sequenceid=409, filesize=30.5 K 2024-11-07T09:37:00,600 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/B/fd1f7ddf65654ac9965fcc5d5d2c1a07 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/fd1f7ddf65654ac9965fcc5d5d2c1a07 2024-11-07T09:37:00,604 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/fd1f7ddf65654ac9965fcc5d5d2c1a07, entries=150, sequenceid=409, filesize=12.0 K 2024-11-07T09:37:00,604 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/.tmp/C/f00fb488689a42f4942264fec4734cc1 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/f00fb488689a42f4942264fec4734cc1 2024-11-07T09:37:00,608 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/f00fb488689a42f4942264fec4734cc1, entries=150, sequenceid=409, filesize=12.0 K 2024-11-07T09:37:00,608 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 81b216f4bb32d787c444da8261b91891 in 1661ms, sequenceid=409, compaction requested=false 2024-11-07T09:37:00,609 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/f41795ffda244dceb7871ee1d6f14c41, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/287bfdbe7ce7446c97d55f3bcc8cdbf2, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/1e5cf4a271af48f1b590d6b4b2f874d6, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/0c367ad1dea44e78956743da27f3f767, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/3c8ef269723744dda71418c66bb09a37, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/24b53a4627684be99e8c00822e6f214c, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/a19c6b5148644e78919a17b21d9bfad9, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/8fc5b5bd41ff4409abb99c8777732f72, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/4fd68b55d7cd4678a5aa25735a6c8ef4, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/cdf7c281a874458598e364ea88f18867, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/a8202baf81214d1ba3dc374cc620fd19, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/d442edf2fc87478abebcb331a41ed2ee, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/b960794f3792449e8f5ef8a11cef8185, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/5a0d0e5971424568a54ab43ab8e7beeb, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/a2e9291928d545098ea3c2de7a3c565b, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/0d2ef765f0ac4bb2b46fd92cbe10816d, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/6ef5ab0c44cb474daf3c8fcb782722d2, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/85a324bf2ba6403aa99be47bbb568062, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/3a6716d59a4946f2976e15f1e238a9cd, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/3144bb6601124f0192cfb04c629890b5, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/f6b59c7ad90a4f2281529ad52ebe4aca, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/614b3593b9b5423a9058ec95fb19f1a9, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/0029131742fe444faae6c4df7182f510, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/11fa9417932c4b10936ff129651ea4dc, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/bfe5e744fb65416ab8a426ab37e16c68, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/405591038c6f4220b8123312447d80b7, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/0cdfe8970a344a6aafb7dee200820400] to archive 2024-11-07T09:37:00,610 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T09:37:00,611 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/f41795ffda244dceb7871ee1d6f14c41 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/f41795ffda244dceb7871ee1d6f14c41 2024-11-07T09:37:00,612 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/287bfdbe7ce7446c97d55f3bcc8cdbf2 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/287bfdbe7ce7446c97d55f3bcc8cdbf2 2024-11-07T09:37:00,613 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/1e5cf4a271af48f1b590d6b4b2f874d6 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/1e5cf4a271af48f1b590d6b4b2f874d6 2024-11-07T09:37:00,614 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/0c367ad1dea44e78956743da27f3f767 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/0c367ad1dea44e78956743da27f3f767 2024-11-07T09:37:00,616 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/3c8ef269723744dda71418c66bb09a37 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/3c8ef269723744dda71418c66bb09a37 2024-11-07T09:37:00,617 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/24b53a4627684be99e8c00822e6f214c to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/24b53a4627684be99e8c00822e6f214c 2024-11-07T09:37:00,618 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/a19c6b5148644e78919a17b21d9bfad9 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/a19c6b5148644e78919a17b21d9bfad9 2024-11-07T09:37:00,619 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/8fc5b5bd41ff4409abb99c8777732f72 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/8fc5b5bd41ff4409abb99c8777732f72 2024-11-07T09:37:00,620 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/4fd68b55d7cd4678a5aa25735a6c8ef4 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/4fd68b55d7cd4678a5aa25735a6c8ef4 2024-11-07T09:37:00,620 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/cdf7c281a874458598e364ea88f18867 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/cdf7c281a874458598e364ea88f18867 2024-11-07T09:37:00,621 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/a8202baf81214d1ba3dc374cc620fd19 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/a8202baf81214d1ba3dc374cc620fd19 2024-11-07T09:37:00,622 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/d442edf2fc87478abebcb331a41ed2ee to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/d442edf2fc87478abebcb331a41ed2ee 2024-11-07T09:37:00,623 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/b960794f3792449e8f5ef8a11cef8185 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/b960794f3792449e8f5ef8a11cef8185 2024-11-07T09:37:00,625 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/5a0d0e5971424568a54ab43ab8e7beeb to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/5a0d0e5971424568a54ab43ab8e7beeb 2024-11-07T09:37:00,626 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/a2e9291928d545098ea3c2de7a3c565b to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/a2e9291928d545098ea3c2de7a3c565b 2024-11-07T09:37:00,627 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/0d2ef765f0ac4bb2b46fd92cbe10816d to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/0d2ef765f0ac4bb2b46fd92cbe10816d 2024-11-07T09:37:00,628 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/6ef5ab0c44cb474daf3c8fcb782722d2 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/6ef5ab0c44cb474daf3c8fcb782722d2 2024-11-07T09:37:00,630 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/85a324bf2ba6403aa99be47bbb568062 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/85a324bf2ba6403aa99be47bbb568062 2024-11-07T09:37:00,631 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/3a6716d59a4946f2976e15f1e238a9cd to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/3a6716d59a4946f2976e15f1e238a9cd 2024-11-07T09:37:00,633 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/3144bb6601124f0192cfb04c629890b5 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/3144bb6601124f0192cfb04c629890b5 2024-11-07T09:37:00,634 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/f6b59c7ad90a4f2281529ad52ebe4aca to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/f6b59c7ad90a4f2281529ad52ebe4aca 2024-11-07T09:37:00,635 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/614b3593b9b5423a9058ec95fb19f1a9 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/614b3593b9b5423a9058ec95fb19f1a9 2024-11-07T09:37:00,636 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/0029131742fe444faae6c4df7182f510 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/0029131742fe444faae6c4df7182f510 2024-11-07T09:37:00,637 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/11fa9417932c4b10936ff129651ea4dc to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/11fa9417932c4b10936ff129651ea4dc 2024-11-07T09:37:00,639 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/bfe5e744fb65416ab8a426ab37e16c68 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/bfe5e744fb65416ab8a426ab37e16c68 2024-11-07T09:37:00,640 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/405591038c6f4220b8123312447d80b7 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/405591038c6f4220b8123312447d80b7 2024-11-07T09:37:00,641 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/0cdfe8970a344a6aafb7dee200820400 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/0cdfe8970a344a6aafb7dee200820400 2024-11-07T09:37:00,643 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/5d5aea263ee84112bb451e1fcbb5ea18, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/0ca5866ee0d34230b06b405614e2ef45, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/23fe023d822342f7a7c73cc0aaa140a1, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/82eb2e0b0c074c49a4dfc03998308af2, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/b2ef66f1ceec4c778433d762171d1c3b, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/d0efd62a5a204250bd84425a73c9d9f6, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/8a938028424d4b3d8def87f501151cf4, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/19b7869e32f84344a9c082a316a8479c, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/99b30512f91c448996adb7f004a9f8b4, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/1cbde7ee907d4aff9751fe9f838874f3, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/5c2d7e23566b4b8488af3fb6f8f9aceb, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/0ca1a05a3d3d420b899158d593839555, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/17fde19135994dfba93aa562331f120e, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/06b79f12538340e2b1ea2b70cfb0ef05, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/a28dd263201f4b97bf76dd7130c4e730, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/6ad46c6bd12940b4a51b6fcb849dd523, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/97caa18626f4400cb0a95b5d76682cf5, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/a46b734b757b479099e69792d028cfd6, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/cd7f6ed3b7ce45c2b6f9a8d8d44ad738, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/b56c79bca4324917a98823c47afb4c7e, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/e03afeea4f1b4447b2808f39fcf3727e, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/ddf6b87ea3d44cd3874b541c057e900b, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/d5b5cf47de854e09ae8aecd980df45eb, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/64be9c2090e44425b7a37db26d73cf79, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/9b1838f1d8cf457f8572e8c0129ce706, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/c6f29b4465564b0e9a8c27387b8982bd, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/ed678bee40b642b393f5292fd41bd825] to archive 2024-11-07T09:37:00,644 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T09:37:00,646 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/5d5aea263ee84112bb451e1fcbb5ea18 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/5d5aea263ee84112bb451e1fcbb5ea18 2024-11-07T09:37:00,647 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/0ca5866ee0d34230b06b405614e2ef45 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/0ca5866ee0d34230b06b405614e2ef45 2024-11-07T09:37:00,648 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/23fe023d822342f7a7c73cc0aaa140a1 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/23fe023d822342f7a7c73cc0aaa140a1 2024-11-07T09:37:00,649 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/82eb2e0b0c074c49a4dfc03998308af2 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/82eb2e0b0c074c49a4dfc03998308af2 2024-11-07T09:37:00,650 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/b2ef66f1ceec4c778433d762171d1c3b to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/b2ef66f1ceec4c778433d762171d1c3b 2024-11-07T09:37:00,651 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/d0efd62a5a204250bd84425a73c9d9f6 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/d0efd62a5a204250bd84425a73c9d9f6 2024-11-07T09:37:00,652 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/8a938028424d4b3d8def87f501151cf4 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/8a938028424d4b3d8def87f501151cf4 2024-11-07T09:37:00,653 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/19b7869e32f84344a9c082a316a8479c to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/19b7869e32f84344a9c082a316a8479c 2024-11-07T09:37:00,653 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/99b30512f91c448996adb7f004a9f8b4 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/99b30512f91c448996adb7f004a9f8b4 2024-11-07T09:37:00,654 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/1cbde7ee907d4aff9751fe9f838874f3 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/1cbde7ee907d4aff9751fe9f838874f3 2024-11-07T09:37:00,655 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/5c2d7e23566b4b8488af3fb6f8f9aceb to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/5c2d7e23566b4b8488af3fb6f8f9aceb 2024-11-07T09:37:00,656 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/0ca1a05a3d3d420b899158d593839555 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/0ca1a05a3d3d420b899158d593839555 2024-11-07T09:37:00,657 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/17fde19135994dfba93aa562331f120e to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/17fde19135994dfba93aa562331f120e 2024-11-07T09:37:00,658 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/06b79f12538340e2b1ea2b70cfb0ef05 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/06b79f12538340e2b1ea2b70cfb0ef05 2024-11-07T09:37:00,659 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/a28dd263201f4b97bf76dd7130c4e730 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/a28dd263201f4b97bf76dd7130c4e730 2024-11-07T09:37:00,660 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/6ad46c6bd12940b4a51b6fcb849dd523 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/6ad46c6bd12940b4a51b6fcb849dd523 2024-11-07T09:37:00,661 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/97caa18626f4400cb0a95b5d76682cf5 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/97caa18626f4400cb0a95b5d76682cf5 2024-11-07T09:37:00,662 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/a46b734b757b479099e69792d028cfd6 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/a46b734b757b479099e69792d028cfd6 2024-11-07T09:37:00,663 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/cd7f6ed3b7ce45c2b6f9a8d8d44ad738 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/cd7f6ed3b7ce45c2b6f9a8d8d44ad738 2024-11-07T09:37:00,664 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/b56c79bca4324917a98823c47afb4c7e to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/b56c79bca4324917a98823c47afb4c7e 2024-11-07T09:37:00,664 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/e03afeea4f1b4447b2808f39fcf3727e to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/e03afeea4f1b4447b2808f39fcf3727e 2024-11-07T09:37:00,665 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/ddf6b87ea3d44cd3874b541c057e900b to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/ddf6b87ea3d44cd3874b541c057e900b 2024-11-07T09:37:00,666 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/d5b5cf47de854e09ae8aecd980df45eb to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/d5b5cf47de854e09ae8aecd980df45eb 2024-11-07T09:37:00,667 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/64be9c2090e44425b7a37db26d73cf79 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/64be9c2090e44425b7a37db26d73cf79 2024-11-07T09:37:00,668 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/9b1838f1d8cf457f8572e8c0129ce706 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/9b1838f1d8cf457f8572e8c0129ce706 2024-11-07T09:37:00,668 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/c6f29b4465564b0e9a8c27387b8982bd to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/c6f29b4465564b0e9a8c27387b8982bd 2024-11-07T09:37:00,669 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/ed678bee40b642b393f5292fd41bd825 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/ed678bee40b642b393f5292fd41bd825 2024-11-07T09:37:00,671 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/956d40ed423c4871aad9f07422e6709e, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/c2bc84f0e2b34f20848921d0a69d3487, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/a4f7741fa20b48f09130f9a4cf8f7c25, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/71a2cea874a0484a81e0d8eae5dca57e, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/e0d3ad31220c4902a7f129210a576710, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/bd73294e814841ec8a5d32e285d44e84, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/c2849a1bc35544a99db9d96bff0d909d, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/6df39d9e04834ea99ca6de2fc2c9f7b9, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/25ab72fa59724b6dae8eb1082fbcf476, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/ed717274184c4321ad515ea731136aad, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/59cc9f550b9240639206c48fa32961fd, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/d1f07bb00cff4f50967dd31744082af7, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/d75a732dded94419b5a00ca04b511ddc, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/fce0e8511c3742bea0b58c0b4df4db52, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/58e613423a2040e8b9677bdd390f31e5, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/6bb0d45573884ef9a4c75dfc6d8f7e5d, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/cea1ffdc225f4259801927af4c836705, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/cf2bd8fc1343428494f209991176bc61, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/408df9aeaae54fd6a6799c65415a408b, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/98b5a61ea61b483e88d068e4f148e0e3, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/7223d81b026249f0a95bb35f7e078c9a, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/9b24b643d93a4b30b986c7e444ceb70a, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/c560e4b8f1a94731acd5b192ddf8382b, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/0922d9b6cc71442a993903011433504e, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/61fb62057b864724acc18f17f2333415, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/6f989abcf8624df99a83942dd4fa43fc, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/f14ff173a7d84c0497924fa9425bb4a6] to archive 2024-11-07T09:37:00,671 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T09:37:00,672 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/956d40ed423c4871aad9f07422e6709e to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/956d40ed423c4871aad9f07422e6709e 2024-11-07T09:37:00,673 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/c2bc84f0e2b34f20848921d0a69d3487 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/c2bc84f0e2b34f20848921d0a69d3487 2024-11-07T09:37:00,674 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/a4f7741fa20b48f09130f9a4cf8f7c25 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/a4f7741fa20b48f09130f9a4cf8f7c25 2024-11-07T09:37:00,675 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/71a2cea874a0484a81e0d8eae5dca57e to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/71a2cea874a0484a81e0d8eae5dca57e 2024-11-07T09:37:00,676 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/e0d3ad31220c4902a7f129210a576710 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/e0d3ad31220c4902a7f129210a576710 2024-11-07T09:37:00,677 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/bd73294e814841ec8a5d32e285d44e84 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/bd73294e814841ec8a5d32e285d44e84 2024-11-07T09:37:00,678 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/c2849a1bc35544a99db9d96bff0d909d to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/c2849a1bc35544a99db9d96bff0d909d 2024-11-07T09:37:00,678 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/6df39d9e04834ea99ca6de2fc2c9f7b9 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/6df39d9e04834ea99ca6de2fc2c9f7b9 2024-11-07T09:37:00,679 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/25ab72fa59724b6dae8eb1082fbcf476 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/25ab72fa59724b6dae8eb1082fbcf476 2024-11-07T09:37:00,680 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/ed717274184c4321ad515ea731136aad to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/ed717274184c4321ad515ea731136aad 2024-11-07T09:37:00,681 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/59cc9f550b9240639206c48fa32961fd to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/59cc9f550b9240639206c48fa32961fd 2024-11-07T09:37:00,682 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/d1f07bb00cff4f50967dd31744082af7 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/d1f07bb00cff4f50967dd31744082af7 2024-11-07T09:37:00,682 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/d75a732dded94419b5a00ca04b511ddc to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/d75a732dded94419b5a00ca04b511ddc 2024-11-07T09:37:00,684 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/fce0e8511c3742bea0b58c0b4df4db52 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/fce0e8511c3742bea0b58c0b4df4db52 2024-11-07T09:37:00,685 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/58e613423a2040e8b9677bdd390f31e5 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/58e613423a2040e8b9677bdd390f31e5 2024-11-07T09:37:00,685 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/6bb0d45573884ef9a4c75dfc6d8f7e5d to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/6bb0d45573884ef9a4c75dfc6d8f7e5d 2024-11-07T09:37:00,687 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/cea1ffdc225f4259801927af4c836705 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/cea1ffdc225f4259801927af4c836705 2024-11-07T09:37:00,687 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/cf2bd8fc1343428494f209991176bc61 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/cf2bd8fc1343428494f209991176bc61 2024-11-07T09:37:00,688 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/408df9aeaae54fd6a6799c65415a408b to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/408df9aeaae54fd6a6799c65415a408b 2024-11-07T09:37:00,689 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/98b5a61ea61b483e88d068e4f148e0e3 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/98b5a61ea61b483e88d068e4f148e0e3 2024-11-07T09:37:00,690 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/7223d81b026249f0a95bb35f7e078c9a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/7223d81b026249f0a95bb35f7e078c9a 2024-11-07T09:37:00,691 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/9b24b643d93a4b30b986c7e444ceb70a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/9b24b643d93a4b30b986c7e444ceb70a 2024-11-07T09:37:00,692 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/c560e4b8f1a94731acd5b192ddf8382b to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/c560e4b8f1a94731acd5b192ddf8382b 2024-11-07T09:37:00,693 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/0922d9b6cc71442a993903011433504e to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/0922d9b6cc71442a993903011433504e 2024-11-07T09:37:00,694 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/61fb62057b864724acc18f17f2333415 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/61fb62057b864724acc18f17f2333415 2024-11-07T09:37:00,695 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/6f989abcf8624df99a83942dd4fa43fc to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/6f989abcf8624df99a83942dd4fa43fc 2024-11-07T09:37:00,695 DEBUG [StoreCloser-TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/f14ff173a7d84c0497924fa9425bb4a6 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/f14ff173a7d84c0497924fa9425bb4a6 2024-11-07T09:37:00,700 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/recovered.edits/412.seqid, newMaxSeqId=412, maxSeqId=4 2024-11-07T09:37:00,700 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. 2024-11-07T09:37:00,701 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1676): Region close journal for 81b216f4bb32d787c444da8261b91891: Waiting for close lock at 1730972218947Running coprocessor pre-close hooks at 1730972218947Disabling compacts and flushes for region at 1730972218947Disabling writes for close at 1730972218947Obtaining lock to block concurrent updates at 1730972218947Preparing flush snapshotting stores in 81b216f4bb32d787c444da8261b91891 at 1730972218947Finished memstore snapshotting TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1730972218948 (+1 ms)Flushing stores of TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891. at 1730972218948Flushing 81b216f4bb32d787c444da8261b91891/A: creating writer at 1730972218948Flushing mob file 81b216f4bb32d787c444da8261b91891/A: appending metadata at 1730972218954 (+6 ms)Flushing mob file 81b216f4bb32d787c444da8261b91891/A: closing flushed file at 1730972218954Flushing 81b216f4bb32d787c444da8261b91891/A: appending metadata at 1730972219365 (+411 ms)Flushing 81b216f4bb32d787c444da8261b91891/A: closing flushed file at 1730972219365Flushing 81b216f4bb32d787c444da8261b91891/B: creating writer at 1730972219773 (+408 ms)Flushing 81b216f4bb32d787c444da8261b91891/B: appending metadata at 1730972219776 (+3 ms)Flushing 81b216f4bb32d787c444da8261b91891/B: closing flushed file at 1730972219776Flushing 81b216f4bb32d787c444da8261b91891/C: creating writer at 1730972220184 (+408 ms)Flushing 81b216f4bb32d787c444da8261b91891/C: appending metadata at 1730972220187 (+3 ms)Flushing 81b216f4bb32d787c444da8261b91891/C: closing flushed file at 1730972220187Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1bf7c6e6: reopening flushed file at 1730972220595 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@205e808b: reopening flushed file at 1730972220600 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@685db540: reopening flushed file at 1730972220604 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 81b216f4bb32d787c444da8261b91891 in 1661ms, sequenceid=409, compaction requested=false at 1730972220608 (+4 ms)Writing region close event to WAL at 1730972220697 (+89 ms)Running coprocessor post-close hooks at 1730972220700 (+3 ms)Closed at 1730972220700 2024-11-07T09:37:00,702 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(157): Closed 81b216f4bb32d787c444da8261b91891 2024-11-07T09:37:00,703 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=81b216f4bb32d787c444da8261b91891, regionState=CLOSED 2024-11-07T09:37:00,704 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; CloseRegionProcedure 81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 because future has completed 2024-11-07T09:37:00,706 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-07T09:37:00,706 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; CloseRegionProcedure 81b216f4bb32d787c444da8261b91891, server=9f1eb259a29b,42895,1730972153970 in 1.9100 sec 2024-11-07T09:37:00,708 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-07T09:37:00,708 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=81b216f4bb32d787c444da8261b91891, UNASSIGN in 1.9140 sec 2024-11-07T09:37:00,709 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-07T09:37:00,710 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9170 sec 2024-11-07T09:37:00,711 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730972220710"}]},"ts":"1730972220710"} 2024-11-07T09:37:00,712 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-07T09:37:00,712 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-07T09:37:00,713 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9260 sec 2024-11-07T09:37:00,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-07T09:37:00,933 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:37:00,934 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-07T09:37:00,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T09:37:00,935 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=66, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T09:37:00,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-07T09:37:00,936 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=66, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T09:37:00,938 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891 2024-11-07T09:37:00,941 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A, FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B, FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C, FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/recovered.edits] 2024-11-07T09:37:00,943 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/23579488d92346fe88edf9a0cf0fdf48 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/23579488d92346fe88edf9a0cf0fdf48 2024-11-07T09:37:00,944 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/aa0e0068d5544e24a8955f72cb7944bd to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/A/aa0e0068d5544e24a8955f72cb7944bd 2024-11-07T09:37:00,946 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/638ccf7f058d4a5ab77ae255955ed1c9 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/638ccf7f058d4a5ab77ae255955ed1c9 2024-11-07T09:37:00,947 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/fd1f7ddf65654ac9965fcc5d5d2c1a07 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/B/fd1f7ddf65654ac9965fcc5d5d2c1a07 2024-11-07T09:37:00,949 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/9cba5593f4de4f32905ab9a44b2ef09d to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/9cba5593f4de4f32905ab9a44b2ef09d 2024-11-07T09:37:00,950 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/f00fb488689a42f4942264fec4734cc1 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/C/f00fb488689a42f4942264fec4734cc1 2024-11-07T09:37:00,953 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/recovered.edits/412.seqid to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891/recovered.edits/412.seqid 2024-11-07T09:37:00,953 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/81b216f4bb32d787c444da8261b91891 2024-11-07T09:37:00,953 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-07T09:37:00,954 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-07T09:37:00,955 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-07T09:37:00,958 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110710592d18cba24af1a23fea1fe1a31512_81b216f4bb32d787c444da8261b91891 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110710592d18cba24af1a23fea1fe1a31512_81b216f4bb32d787c444da8261b91891 2024-11-07T09:37:00,959 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110714ef3a86f1a24b4bb0359a26d0c33432_81b216f4bb32d787c444da8261b91891 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110714ef3a86f1a24b4bb0359a26d0c33432_81b216f4bb32d787c444da8261b91891 2024-11-07T09:37:00,961 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411071b1a7d6af3d54283a7ff42cc9642dc7d_81b216f4bb32d787c444da8261b91891 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411071b1a7d6af3d54283a7ff42cc9642dc7d_81b216f4bb32d787c444da8261b91891 2024-11-07T09:37:00,962 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110721d85f62bf45441c9c730cbf6dc28c62_81b216f4bb32d787c444da8261b91891 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110721d85f62bf45441c9c730cbf6dc28c62_81b216f4bb32d787c444da8261b91891 2024-11-07T09:37:00,963 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110743a1c59907954c989974af6fd3364b39_81b216f4bb32d787c444da8261b91891 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110743a1c59907954c989974af6fd3364b39_81b216f4bb32d787c444da8261b91891 2024-11-07T09:37:00,964 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107455d4edc4dfb45418d83b14402255612_81b216f4bb32d787c444da8261b91891 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107455d4edc4dfb45418d83b14402255612_81b216f4bb32d787c444da8261b91891 2024-11-07T09:37:00,965 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110749d1d1714cb842bd92ef6cd10dc33f2e_81b216f4bb32d787c444da8261b91891 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110749d1d1714cb842bd92ef6cd10dc33f2e_81b216f4bb32d787c444da8261b91891 2024-11-07T09:37:00,966 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107531172a5ed28455297081ff50e6fd164_81b216f4bb32d787c444da8261b91891 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107531172a5ed28455297081ff50e6fd164_81b216f4bb32d787c444da8261b91891 2024-11-07T09:37:00,967 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107668ef98ebbc64606ada2714aa545130b_81b216f4bb32d787c444da8261b91891 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107668ef98ebbc64606ada2714aa545130b_81b216f4bb32d787c444da8261b91891 2024-11-07T09:37:00,968 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110770ede35b35b04f9facd1100dd0b75010_81b216f4bb32d787c444da8261b91891 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110770ede35b35b04f9facd1100dd0b75010_81b216f4bb32d787c444da8261b91891 2024-11-07T09:37:00,970 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411079df2a4dd6c6e4fab891ccc186ffac383_81b216f4bb32d787c444da8261b91891 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411079df2a4dd6c6e4fab891ccc186ffac383_81b216f4bb32d787c444da8261b91891 2024-11-07T09:37:00,971 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107bfe87bb00bad4d46a690ee3ab08ec8ac_81b216f4bb32d787c444da8261b91891 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107bfe87bb00bad4d46a690ee3ab08ec8ac_81b216f4bb32d787c444da8261b91891 2024-11-07T09:37:00,972 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107c0ad0cd2b4894bbc9d105ad91abe4e38_81b216f4bb32d787c444da8261b91891 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107c0ad0cd2b4894bbc9d105ad91abe4e38_81b216f4bb32d787c444da8261b91891 2024-11-07T09:37:00,973 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107c438d53b4b6648a1a4e2f7dce5bbc856_81b216f4bb32d787c444da8261b91891 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107c438d53b4b6648a1a4e2f7dce5bbc856_81b216f4bb32d787c444da8261b91891 2024-11-07T09:37:00,975 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107c5f6f095f5a94831966feb96b666ad0c_81b216f4bb32d787c444da8261b91891 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107c5f6f095f5a94831966feb96b666ad0c_81b216f4bb32d787c444da8261b91891 2024-11-07T09:37:00,976 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107c9ad9c0732d449b2ad71d06ba94047a1_81b216f4bb32d787c444da8261b91891 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107c9ad9c0732d449b2ad71d06ba94047a1_81b216f4bb32d787c444da8261b91891 2024-11-07T09:37:00,977 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107ca23ad9485d74f9bad5981ff256f462c_81b216f4bb32d787c444da8261b91891 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107ca23ad9485d74f9bad5981ff256f462c_81b216f4bb32d787c444da8261b91891 2024-11-07T09:37:00,978 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107cb64dc0201894a25b67c7d9e961b120e_81b216f4bb32d787c444da8261b91891 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107cb64dc0201894a25b67c7d9e961b120e_81b216f4bb32d787c444da8261b91891 2024-11-07T09:37:00,979 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107da66a09f3b2349de9d1dee53bb530780_81b216f4bb32d787c444da8261b91891 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107da66a09f3b2349de9d1dee53bb530780_81b216f4bb32d787c444da8261b91891 2024-11-07T09:37:00,980 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107e2cc54ff980f43d9899c3db2db091ad5_81b216f4bb32d787c444da8261b91891 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107e2cc54ff980f43d9899c3db2db091ad5_81b216f4bb32d787c444da8261b91891 2024-11-07T09:37:00,981 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f829067f54ae474bbef7f068a0962a9a_81b216f4bb32d787c444da8261b91891 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f829067f54ae474bbef7f068a0962a9a_81b216f4bb32d787c444da8261b91891 2024-11-07T09:37:00,982 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-07T09:37:00,984 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=66, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T09:37:00,986 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-07T09:37:00,988 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-07T09:37:00,989 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=66, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T09:37:00,989 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-07T09:37:00,989 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1730972220989"}]},"ts":"9223372036854775807"} 2024-11-07T09:37:00,991 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-07T09:37:00,991 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 81b216f4bb32d787c444da8261b91891, NAME => 'TestAcidGuarantees,,1730972185977.81b216f4bb32d787c444da8261b91891.', STARTKEY => '', ENDKEY => ''}] 2024-11-07T09:37:00,991 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-07T09:37:00,991 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1730972220991"}]},"ts":"9223372036854775807"} 2024-11-07T09:37:00,993 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-07T09:37:00,993 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=66, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T09:37:00,994 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 59 msec 2024-11-07T09:37:01,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-07T09:37:01,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-07T09:37:01,043 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:37:01,052 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=238 (was 238), OpenFileDescriptor=453 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=385 (was 391), ProcessCount=11 (was 11), AvailableMemoryMB=4726 (was 4898) 2024-11-07T09:37:01,060 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=238, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=385, ProcessCount=11, AvailableMemoryMB=4725 2024-11-07T09:37:01,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T09:37:01,061 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T09:37:01,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=67, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-07T09:37:01,063 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T09:37:01,063 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:01,063 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 67 2024-11-07T09:37:01,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-07T09:37:01,064 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T09:37:01,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742079_1255 (size=959) 2024-11-07T09:37:01,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-07T09:37:01,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-07T09:37:01,471 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260 2024-11-07T09:37:01,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742080_1256 (size=53) 2024-11-07T09:37:01,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-07T09:37:01,878 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T09:37:01,878 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 5165da7af32dbdfd119fd1fa7507f91f, disabling compactions & flushes 2024-11-07T09:37:01,878 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:01,878 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:01,878 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. after waiting 0 ms 2024-11-07T09:37:01,878 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:01,878 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:01,878 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5165da7af32dbdfd119fd1fa7507f91f: Waiting for close lock at 1730972221878Disabling compacts and flushes for region at 1730972221878Disabling writes for close at 1730972221878Writing region close event to WAL at 1730972221878Closed at 1730972221878 2024-11-07T09:37:01,879 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T09:37:01,880 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1730972221879"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730972221879"}]},"ts":"1730972221879"} 2024-11-07T09:37:01,881 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-07T09:37:01,882 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T09:37:01,882 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730972221882"}]},"ts":"1730972221882"} 2024-11-07T09:37:01,883 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-07T09:37:01,883 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5165da7af32dbdfd119fd1fa7507f91f, ASSIGN}] 2024-11-07T09:37:01,884 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5165da7af32dbdfd119fd1fa7507f91f, ASSIGN 2024-11-07T09:37:01,884 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5165da7af32dbdfd119fd1fa7507f91f, ASSIGN; state=OFFLINE, location=9f1eb259a29b,42895,1730972153970; forceNewPlan=false, retain=false 2024-11-07T09:37:02,035 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=5165da7af32dbdfd119fd1fa7507f91f, regionState=OPENING, regionLocation=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:02,037 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5165da7af32dbdfd119fd1fa7507f91f, ASSIGN because future has completed 2024-11-07T09:37:02,037 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970}] 2024-11-07T09:37:02,191 INFO [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:02,191 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(7752): Opening region: {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} 2024-11-07T09:37:02,192 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5165da7af32dbdfd119fd1fa7507f91f 2024-11-07T09:37:02,192 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T09:37:02,192 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(7794): checking encryption for 5165da7af32dbdfd119fd1fa7507f91f 2024-11-07T09:37:02,192 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(7797): checking classloading for 5165da7af32dbdfd119fd1fa7507f91f 2024-11-07T09:37:02,193 INFO [StoreOpener-5165da7af32dbdfd119fd1fa7507f91f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5165da7af32dbdfd119fd1fa7507f91f 2024-11-07T09:37:02,194 INFO [StoreOpener-5165da7af32dbdfd119fd1fa7507f91f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5165da7af32dbdfd119fd1fa7507f91f columnFamilyName A 2024-11-07T09:37:02,194 DEBUG [StoreOpener-5165da7af32dbdfd119fd1fa7507f91f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:02,195 INFO [StoreOpener-5165da7af32dbdfd119fd1fa7507f91f-1 {}] regionserver.HStore(327): Store=5165da7af32dbdfd119fd1fa7507f91f/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T09:37:02,195 INFO [StoreOpener-5165da7af32dbdfd119fd1fa7507f91f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5165da7af32dbdfd119fd1fa7507f91f 2024-11-07T09:37:02,195 INFO [StoreOpener-5165da7af32dbdfd119fd1fa7507f91f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5165da7af32dbdfd119fd1fa7507f91f columnFamilyName B 2024-11-07T09:37:02,196 DEBUG [StoreOpener-5165da7af32dbdfd119fd1fa7507f91f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:02,196 INFO [StoreOpener-5165da7af32dbdfd119fd1fa7507f91f-1 {}] regionserver.HStore(327): Store=5165da7af32dbdfd119fd1fa7507f91f/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T09:37:02,196 INFO [StoreOpener-5165da7af32dbdfd119fd1fa7507f91f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5165da7af32dbdfd119fd1fa7507f91f 2024-11-07T09:37:02,197 INFO [StoreOpener-5165da7af32dbdfd119fd1fa7507f91f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5165da7af32dbdfd119fd1fa7507f91f columnFamilyName C 2024-11-07T09:37:02,197 DEBUG [StoreOpener-5165da7af32dbdfd119fd1fa7507f91f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:02,197 INFO [StoreOpener-5165da7af32dbdfd119fd1fa7507f91f-1 {}] regionserver.HStore(327): Store=5165da7af32dbdfd119fd1fa7507f91f/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T09:37:02,197 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1038): replaying wal for 5165da7af32dbdfd119fd1fa7507f91f 2024-11-07T09:37:02,198 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f 2024-11-07T09:37:02,198 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f 2024-11-07T09:37:02,199 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1048): stopping wal replay for 5165da7af32dbdfd119fd1fa7507f91f 2024-11-07T09:37:02,199 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1060): Cleaning up temporary data for 5165da7af32dbdfd119fd1fa7507f91f 2024-11-07T09:37:02,200 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T09:37:02,201 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1093): writing seq id for 5165da7af32dbdfd119fd1fa7507f91f 2024-11-07T09:37:02,202 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T09:37:02,203 INFO [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1114): Opened 5165da7af32dbdfd119fd1fa7507f91f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64784238, jitterRate=-0.0346396267414093}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T09:37:02,203 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5165da7af32dbdfd119fd1fa7507f91f 2024-11-07T09:37:02,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-07T09:37:02,203 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1006): Region open journal for 5165da7af32dbdfd119fd1fa7507f91f: Running coprocessor pre-open hook at 1730972222192Writing region info on filesystem at 1730972222192Initializing all the Stores at 1730972222192Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730972222193 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730972222193Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730972222193Cleaning up temporary data from old regions at 1730972222199 (+6 ms)Running coprocessor post-open hooks at 1730972222203 (+4 ms)Region opened successfully at 1730972222203 2024-11-07T09:37:02,204 INFO [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., pid=69, masterSystemTime=1730972222188 2024-11-07T09:37:02,205 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:02,205 INFO [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:02,206 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=5165da7af32dbdfd119fd1fa7507f91f, regionState=OPEN, openSeqNum=2, regionLocation=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:02,207 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 because future has completed 2024-11-07T09:37:02,209 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-07T09:37:02,209 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; OpenRegionProcedure 5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 in 171 msec 2024-11-07T09:37:02,210 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=68, resume processing ppid=67 2024-11-07T09:37:02,210 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, ppid=67, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5165da7af32dbdfd119fd1fa7507f91f, ASSIGN in 326 msec 2024-11-07T09:37:02,211 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T09:37:02,211 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730972222211"}]},"ts":"1730972222211"} 2024-11-07T09:37:02,212 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-07T09:37:02,213 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T09:37:02,214 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1520 sec 2024-11-07T09:37:03,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-07T09:37:03,213 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:37:03,214 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bdc5a1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:03,214 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:37:03,214 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:37:03,215 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:37:03,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:37:03,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:37:03,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5259b546, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:03,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:37:03,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:37:03,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:03,217 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46552, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:37:03,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d20c4e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:03,218 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:37:03,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:37:03,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:37:03,219 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42818, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:37:03,220 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T09:37:03,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 9f1eb259a29b,41689,1730972153310 2024-11-07T09:37:03,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5e67ae44 2024-11-07T09:37:03,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T09:37:03,221 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46560, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T09:37:03,222 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20832dd2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:03,222 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:37:03,222 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:37:03,223 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:37:03,223 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:37:03,223 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:37:03,223 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a38f1f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:03,223 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:37:03,223 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:37:03,224 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:03,224 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:37:03,225 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@485126d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:03,225 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4eb66027, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:03,225 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:37:03,225 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:37:03,226 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:37:03,226 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:37:03,226 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:37:03,226 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24a5ebd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:03,226 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:37:03,226 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:37:03,227 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:03,227 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46598, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:37:03,228 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ed7d1d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:03,228 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68b4a085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:03,228 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:37:03,228 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:37:03,229 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:37:03,229 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:37:03,229 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:37:03,229 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bc31512, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:03,229 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:37:03,229 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:37:03,229 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:03,230 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46610, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:37:03,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27ef20d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:03,231 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75f8f105, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:03,231 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:37:03,231 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:37:03,231 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:37:03,231 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:37:03,231 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:37:03,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68dfbdce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:03,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:37:03,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:37:03,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:03,232 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46632, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:37:03,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@629af9bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:03,233 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@aaa706, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:03,233 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:37:03,233 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:37:03,234 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:37:03,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:37:03,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:37:03,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b02cafd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:03,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:37:03,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:37:03,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:03,235 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46656, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:37:03,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@636baedf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:03,236 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@417b74fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:03,236 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:37:03,236 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:37:03,236 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:37:03,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:37:03,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:37:03,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4634bca5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:03,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:37:03,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:37:03,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:03,238 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46674, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:37:03,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fd0d5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:03,238 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e0a17bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:03,239 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:37:03,239 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:37:03,239 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:37:03,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:37:03,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:37:03,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b47d5d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:03,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:37:03,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:37:03,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:03,240 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46694, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:37:03,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e3fe818, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:03,241 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bb595da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:03,241 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:37:03,241 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:37:03,242 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:37:03,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:37:03,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:37:03,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18df6adf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:03,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:37:03,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:37:03,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:03,243 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46710, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:37:03,244 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e57f4b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:03,244 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@262694dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:03,244 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:37:03,244 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:37:03,245 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:37:03,245 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:37:03,245 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:37:03,245 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69812200, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:03,245 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:37:03,245 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:37:03,246 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:03,246 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46728, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:37:03,247 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16e2ed2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:03,247 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@df95348, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:03,247 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:37:03,247 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:37:03,248 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:37:03,248 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:37:03,248 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:37:03,248 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@374d693, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:03,248 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:37:03,248 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:37:03,249 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:03,249 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46744, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:37:03,250 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4eb4235a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:03,250 DEBUG [Thread-1190 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:37:03,251 DEBUG [Thread-1199 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:37:03,251 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T09:37:03,251 DEBUG [Thread-1186 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:37:03,251 DEBUG [Thread-1197 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:37:03,251 DEBUG [Thread-1184 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:37:03,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:37:03,252 DEBUG [Thread-1182 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:37:03,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:37:03,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:37:03,250 DEBUG [Thread-1195 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:37:03,250 DEBUG [Thread-1188 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:37:03,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:37:03,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-07T09:37:03,252 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:37:03,250 DEBUG [Thread-1193 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:37:03,252 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:37:03,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:37:03,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:37:03,253 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42826, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:37:03,253 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T09:37:03,253 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:37:03,253 DEBUG [Thread-1201 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:37:03,253 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:37:03,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:37:03,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:37:03,254 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42842, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:37:03,254 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42856, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:37:03,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:37:03,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:37:03,254 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T09:37:03,254 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T09:37:03,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:37:03,255 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:37:03,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-07T09:37:03,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:37:03,255 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:37:03,255 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:37:03,255 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:37:03,255 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42866, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:37:03,256 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42868, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:37:03,256 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42872, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:37:03,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2] 2024-11-07T09:37:03,256 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42892, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:37:03,256 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42878, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:37:03,257 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42908, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:37:03,257 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42912, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:37:03,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2] 2024-11-07T09:37:03,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2] 2024-11-07T09:37:03,257 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2] 2024-11-07T09:37:03,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2] 2024-11-07T09:37:03,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2] 2024-11-07T09:37:03,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2] 2024-11-07T09:37:03,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2] 2024-11-07T09:37:03,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2] 2024-11-07T09:37:03,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2] 2024-11-07T09:37:03,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 5165da7af32dbdfd119fd1fa7507f91f 2024-11-07T09:37:03,272 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:37:03,272 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5165da7af32dbdfd119fd1fa7507f91f 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-07T09:37:03,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:03,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972283311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:03,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:03,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42842 deadline: 1730972283315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:03,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:03,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:03,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:03,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:03,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:03,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:03,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:03,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42826 deadline: 1730972283316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:03,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:03,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:03,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:03,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:03,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730972283319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:03,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:03,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:03,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:03,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1730972283319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:03,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:03,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:03,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:03,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:03,327 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/f038e89264834b0a8150eac5d1beec01 is 50, key is test_row_0/A:col10/1730972223272/Put/seqid=0 2024-11-07T09:37:03,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742081_1257 (size=12001) 2024-11-07T09:37:03,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-07T09:37:03,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-07T09:37:03,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:03,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. as already flushing 2024-11-07T09:37:03,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:03,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:03,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:03,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:03,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:03,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42842 deadline: 1730972283422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:03,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:03,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:03,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:03,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972283422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:03,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:03,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:03,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:03,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:03,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:03,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42826 deadline: 1730972283433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:03,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:03,436 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:03,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730972283433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:03,436 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:03,436 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:03,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:03,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1730972283433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:03,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:03,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:03,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:03,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:03,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:03,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:03,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-07T09:37:03,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:03,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. as already flushing 2024-11-07T09:37:03,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:03,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:03,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:03,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:03,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-07T09:37:03,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:03,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42842 deadline: 1730972283632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:03,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:03,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972283633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:03,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:03,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:03,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:03,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:03,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:03,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:03,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:03,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42826 deadline: 1730972283643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:03,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:03,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:03,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:03,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:03,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730972283643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:03,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:03,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1730972283643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:03,646 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:03,646 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:03,646 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:03,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:03,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:03,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:03,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-07T09:37:03,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:03,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. as already flushing 2024-11-07T09:37:03,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:03,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:03,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:03,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:03,743 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/f038e89264834b0a8150eac5d1beec01 2024-11-07T09:37:03,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/62366126444b4854a8f65a180a02a184 is 50, key is test_row_0/B:col10/1730972223272/Put/seqid=0 2024-11-07T09:37:03,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742082_1258 (size=12001) 2024-11-07T09:37:03,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-07T09:37:03,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:03,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. as already flushing 2024-11-07T09:37:03,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:03,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:03,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:03,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:03,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-07T09:37:03,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:03,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:03,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42842 deadline: 1730972283943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:03,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972283943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:03,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:03,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:03,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:03,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:03,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:03,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:03,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:03,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42826 deadline: 1730972283953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:03,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:03,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:03,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730972283953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:03,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:03,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:03,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:03,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:03,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:03,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:03,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1730972283953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:03,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:03,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:03,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:04,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-07T09:37:04,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:04,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. as already flushing 2024-11-07T09:37:04,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:04,024 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:04,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:04,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:04,176 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/62366126444b4854a8f65a180a02a184 2024-11-07T09:37:04,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-07T09:37:04,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:04,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. as already flushing 2024-11-07T09:37:04,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:04,178 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:04,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:04,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:04,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/39df6e17262b4f5eae374674548e744d is 50, key is test_row_0/C:col10/1730972223272/Put/seqid=0 2024-11-07T09:37:04,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742083_1259 (size=12001) 2024-11-07T09:37:04,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-07T09:37:04,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:04,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. as already flushing 2024-11-07T09:37:04,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:04,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:04,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:04,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:04,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-07T09:37:04,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:04,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972284452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:04,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:04,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:04,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:04,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:04,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42842 deadline: 1730972284453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:04,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:04,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:04,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:04,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:04,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:04,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42826 deadline: 1730972284463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:04,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730972284463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:04,467 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:04,467 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:04,467 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:04,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:04,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1730972284463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:04,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:04,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:04,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:04,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:04,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:04,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:04,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-07T09:37:04,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:04,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. as already flushing 2024-11-07T09:37:04,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:04,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:04,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:04,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:04,603 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/39df6e17262b4f5eae374674548e744d 2024-11-07T09:37:04,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/f038e89264834b0a8150eac5d1beec01 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/f038e89264834b0a8150eac5d1beec01 2024-11-07T09:37:04,616 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/f038e89264834b0a8150eac5d1beec01, entries=150, sequenceid=14, filesize=11.7 K 2024-11-07T09:37:04,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/62366126444b4854a8f65a180a02a184 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/62366126444b4854a8f65a180a02a184 2024-11-07T09:37:04,622 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/62366126444b4854a8f65a180a02a184, entries=150, sequenceid=14, filesize=11.7 K 2024-11-07T09:37:04,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/39df6e17262b4f5eae374674548e744d as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/39df6e17262b4f5eae374674548e744d 2024-11-07T09:37:04,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/39df6e17262b4f5eae374674548e744d, entries=150, sequenceid=14, filesize=11.7 K 2024-11-07T09:37:04,629 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 5165da7af32dbdfd119fd1fa7507f91f in 1356ms, sequenceid=14, compaction requested=false 2024-11-07T09:37:04,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:04,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-07T09:37:04,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:04,638 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 5165da7af32dbdfd119fd1fa7507f91f 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-07T09:37:04,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/dacc80c57c334f31be4b974b0de66fc6 is 50, key is test_row_0/A:col10/1730972223306/Put/seqid=0 2024-11-07T09:37:04,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742084_1260 (size=12001) 2024-11-07T09:37:05,051 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/dacc80c57c334f31be4b974b0de66fc6 2024-11-07T09:37:05,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/a4d5a2ecbc234470b4556be2cdf0b766 is 50, key is test_row_0/B:col10/1730972223306/Put/seqid=0 2024-11-07T09:37:05,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742085_1261 (size=12001) 2024-11-07T09:37:05,098 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/a4d5a2ecbc234470b4556be2cdf0b766 2024-11-07T09:37:05,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/ad0dd1c0f59641f484f4351e0c2cba40 is 50, key is test_row_0/C:col10/1730972223306/Put/seqid=0 2024-11-07T09:37:05,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742086_1262 (size=12001) 2024-11-07T09:37:05,112 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/ad0dd1c0f59641f484f4351e0c2cba40 2024-11-07T09:37:05,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/dacc80c57c334f31be4b974b0de66fc6 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/dacc80c57c334f31be4b974b0de66fc6 2024-11-07T09:37:05,133 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/dacc80c57c334f31be4b974b0de66fc6, entries=150, sequenceid=37, filesize=11.7 K 2024-11-07T09:37:05,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/a4d5a2ecbc234470b4556be2cdf0b766 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/a4d5a2ecbc234470b4556be2cdf0b766 2024-11-07T09:37:05,139 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/a4d5a2ecbc234470b4556be2cdf0b766, entries=150, sequenceid=37, filesize=11.7 K 2024-11-07T09:37:05,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/ad0dd1c0f59641f484f4351e0c2cba40 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/ad0dd1c0f59641f484f4351e0c2cba40 2024-11-07T09:37:05,146 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/ad0dd1c0f59641f484f4351e0c2cba40, entries=150, sequenceid=37, filesize=11.7 K 2024-11-07T09:37:05,147 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=0 B/0 for 5165da7af32dbdfd119fd1fa7507f91f in 509ms, sequenceid=37, compaction requested=false 2024-11-07T09:37:05,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:05,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:05,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-07T09:37:05,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-07T09:37:05,153 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-07T09:37:05,153 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8950 sec 2024-11-07T09:37:05,157 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.9020 sec 2024-11-07T09:37:05,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-07T09:37:05,403 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:37:05,404 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T09:37:05,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-07T09:37:05,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-07T09:37:05,406 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T09:37:05,407 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T09:37:05,407 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T09:37:05,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 5165da7af32dbdfd119fd1fa7507f91f 2024-11-07T09:37:05,477 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:37:05,477 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5165da7af32dbdfd119fd1fa7507f91f 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-07T09:37:05,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/2b58c0f555904d7e930f1857a61a7d75 is 50, key is test_row_0/A:col10/1730972225474/Put/seqid=0 2024-11-07T09:37:05,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742087_1263 (size=12001) 2024-11-07T09:37:05,503 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/2b58c0f555904d7e930f1857a61a7d75 2024-11-07T09:37:05,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-07T09:37:05,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:05,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1730972285508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:05,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:05,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:05,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:05,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/18cb8f3e32b148708ec3315344bf53a4 is 50, key is test_row_0/B:col10/1730972225474/Put/seqid=0 2024-11-07T09:37:05,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:05,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42842 deadline: 1730972285510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:05,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:05,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:05,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972285510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:05,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:05,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:05,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:05,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:05,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:05,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:05,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:05,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730972285514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:05,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42826 deadline: 1730972285512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:05,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:05,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:05,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:05,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:05,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:05,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:05,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742088_1264 (size=12001) 2024-11-07T09:37:05,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-07T09:37:05,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:05,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. as already flushing 2024-11-07T09:37:05,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:05,562 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:05,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:05,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:05,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:05,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1730972285623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:05,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:05,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:05,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42842 deadline: 1730972285623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:05,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:05,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:05,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:05,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:05,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:05,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730972285623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:05,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:05,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:05,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:05,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:05,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:05,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972285623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:05,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:05,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:05,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42826 deadline: 1730972285623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:05,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:05,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:05,628 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:05,628 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:05,628 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:05,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-07T09:37:05,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:05,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. as already flushing 2024-11-07T09:37:05,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:05,716 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:05,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:05,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:05,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-07T09:37:05,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:05,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:05,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730972285833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:05,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42842 deadline: 1730972285833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:05,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:05,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:05,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:05,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:05,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42826 deadline: 1730972285833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:05,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:05,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:05,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:05,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:05,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:05,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:05,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:05,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:05,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1730972285833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:05,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972285833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:05,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:05,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:05,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:05,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:05,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:05,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:05,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-07T09:37:05,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:05,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. as already flushing 2024-11-07T09:37:05,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:05,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:05,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:05,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:05,931 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/18cb8f3e32b148708ec3315344bf53a4 2024-11-07T09:37:05,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/4884e1731666416aa699d4f36d08e8de is 50, key is test_row_0/C:col10/1730972225474/Put/seqid=0 2024-11-07T09:37:05,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742089_1265 (size=12001) 2024-11-07T09:37:05,955 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/4884e1731666416aa699d4f36d08e8de 2024-11-07T09:37:05,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/2b58c0f555904d7e930f1857a61a7d75 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/2b58c0f555904d7e930f1857a61a7d75 2024-11-07T09:37:05,968 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/2b58c0f555904d7e930f1857a61a7d75, entries=150, sequenceid=49, filesize=11.7 K 2024-11-07T09:37:05,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/18cb8f3e32b148708ec3315344bf53a4 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/18cb8f3e32b148708ec3315344bf53a4 2024-11-07T09:37:05,976 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/18cb8f3e32b148708ec3315344bf53a4, entries=150, sequenceid=49, filesize=11.7 K 2024-11-07T09:37:05,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/4884e1731666416aa699d4f36d08e8de as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/4884e1731666416aa699d4f36d08e8de 2024-11-07T09:37:05,982 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/4884e1731666416aa699d4f36d08e8de, entries=150, sequenceid=49, filesize=11.7 K 2024-11-07T09:37:05,983 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 5165da7af32dbdfd119fd1fa7507f91f in 506ms, sequenceid=49, compaction requested=true 2024-11-07T09:37:05,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:05,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5165da7af32dbdfd119fd1fa7507f91f:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T09:37:05,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:05,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5165da7af32dbdfd119fd1fa7507f91f:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T09:37:05,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:05,983 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:37:05,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5165da7af32dbdfd119fd1fa7507f91f:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:37:05,983 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:37:05,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:37:05,985 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:37:05,985 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 5165da7af32dbdfd119fd1fa7507f91f/A is initiating minor compaction (all files) 2024-11-07T09:37:05,985 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5165da7af32dbdfd119fd1fa7507f91f/A in TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:05,985 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/f038e89264834b0a8150eac5d1beec01, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/dacc80c57c334f31be4b974b0de66fc6, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/2b58c0f555904d7e930f1857a61a7d75] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp, totalSize=35.2 K 2024-11-07T09:37:05,985 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:37:05,985 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 5165da7af32dbdfd119fd1fa7507f91f/B is initiating minor compaction (all files) 2024-11-07T09:37:05,985 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5165da7af32dbdfd119fd1fa7507f91f/B in TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:05,985 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/62366126444b4854a8f65a180a02a184, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/a4d5a2ecbc234470b4556be2cdf0b766, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/18cb8f3e32b148708ec3315344bf53a4] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp, totalSize=35.2 K 2024-11-07T09:37:05,986 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 62366126444b4854a8f65a180a02a184, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1730972223268 2024-11-07T09:37:05,987 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting f038e89264834b0a8150eac5d1beec01, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1730972223268 2024-11-07T09:37:05,987 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting a4d5a2ecbc234470b4556be2cdf0b766, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1730972223306 2024-11-07T09:37:05,987 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting dacc80c57c334f31be4b974b0de66fc6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1730972223306 2024-11-07T09:37:05,987 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 18cb8f3e32b148708ec3315344bf53a4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1730972225474 2024-11-07T09:37:05,987 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b58c0f555904d7e930f1857a61a7d75, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1730972225474 2024-11-07T09:37:06,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-07T09:37:06,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:06,024 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 5165da7af32dbdfd119fd1fa7507f91f 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T09:37:06,027 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5165da7af32dbdfd119fd1fa7507f91f#B#compaction#219 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:37:06,028 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/f5a7a611fe294bc49cba015cfc0c3ed3 is 50, key is test_row_0/B:col10/1730972225474/Put/seqid=0 2024-11-07T09:37:06,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-07T09:37:06,034 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5165da7af32dbdfd119fd1fa7507f91f#A#compaction#220 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:37:06,035 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/4aa193391d4a4cbe9d33aba3559a39f7 is 50, key is test_row_0/A:col10/1730972225474/Put/seqid=0 2024-11-07T09:37:06,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/f6903116b51345c8babd0eada216f02d is 50, key is test_row_0/A:col10/1730972225511/Put/seqid=0 2024-11-07T09:37:06,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742090_1266 (size=12104) 2024-11-07T09:37:06,073 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/f5a7a611fe294bc49cba015cfc0c3ed3 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/f5a7a611fe294bc49cba015cfc0c3ed3 2024-11-07T09:37:06,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742091_1267 (size=12104) 2024-11-07T09:37:06,083 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5165da7af32dbdfd119fd1fa7507f91f/B of 5165da7af32dbdfd119fd1fa7507f91f into f5a7a611fe294bc49cba015cfc0c3ed3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:37:06,083 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:06,083 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., storeName=5165da7af32dbdfd119fd1fa7507f91f/B, priority=13, startTime=1730972225983; duration=0sec 2024-11-07T09:37:06,084 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:37:06,084 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5165da7af32dbdfd119fd1fa7507f91f:B 2024-11-07T09:37:06,085 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:37:06,087 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:37:06,087 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 5165da7af32dbdfd119fd1fa7507f91f/C is initiating minor compaction (all files) 2024-11-07T09:37:06,087 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5165da7af32dbdfd119fd1fa7507f91f/C in TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:06,087 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/39df6e17262b4f5eae374674548e744d, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/ad0dd1c0f59641f484f4351e0c2cba40, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/4884e1731666416aa699d4f36d08e8de] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp, totalSize=35.2 K 2024-11-07T09:37:06,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742092_1268 (size=12001) 2024-11-07T09:37:06,088 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 39df6e17262b4f5eae374674548e744d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1730972223268 2024-11-07T09:37:06,088 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting ad0dd1c0f59641f484f4351e0c2cba40, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1730972223306 2024-11-07T09:37:06,089 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/f6903116b51345c8babd0eada216f02d 2024-11-07T09:37:06,090 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 4884e1731666416aa699d4f36d08e8de, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1730972225474 2024-11-07T09:37:06,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/b72c10544cb04b8b9a4263601a34a73a is 50, key is test_row_0/B:col10/1730972225511/Put/seqid=0 2024-11-07T09:37:06,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742093_1269 (size=12001) 2024-11-07T09:37:06,113 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5165da7af32dbdfd119fd1fa7507f91f#C#compaction#223 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:37:06,114 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/6e9bc4bd75bd4ebfbb6e9f8e058d39c8 is 50, key is test_row_0/C:col10/1730972225474/Put/seqid=0 2024-11-07T09:37:06,115 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/b72c10544cb04b8b9a4263601a34a73a 2024-11-07T09:37:06,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742094_1270 (size=12104) 2024-11-07T09:37:06,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/bb1982c8ac434b918cfb128749d9c913 is 50, key is test_row_0/C:col10/1730972225511/Put/seqid=0 2024-11-07T09:37:06,144 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/6e9bc4bd75bd4ebfbb6e9f8e058d39c8 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/6e9bc4bd75bd4ebfbb6e9f8e058d39c8 2024-11-07T09:37:06,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. as already flushing 2024-11-07T09:37:06,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 5165da7af32dbdfd119fd1fa7507f91f 2024-11-07T09:37:06,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742095_1271 (size=12001) 2024-11-07T09:37:06,157 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/bb1982c8ac434b918cfb128749d9c913 2024-11-07T09:37:06,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/f6903116b51345c8babd0eada216f02d as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/f6903116b51345c8babd0eada216f02d 2024-11-07T09:37:06,167 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/f6903116b51345c8babd0eada216f02d, entries=150, sequenceid=73, filesize=11.7 K 2024-11-07T09:37:06,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/b72c10544cb04b8b9a4263601a34a73a as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/b72c10544cb04b8b9a4263601a34a73a 2024-11-07T09:37:06,175 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/b72c10544cb04b8b9a4263601a34a73a, entries=150, sequenceid=73, filesize=11.7 K 2024-11-07T09:37:06,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/bb1982c8ac434b918cfb128749d9c913 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/bb1982c8ac434b918cfb128749d9c913 2024-11-07T09:37:06,181 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/bb1982c8ac434b918cfb128749d9c913, entries=150, sequenceid=73, filesize=11.7 K 2024-11-07T09:37:06,193 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5165da7af32dbdfd119fd1fa7507f91f/C of 5165da7af32dbdfd119fd1fa7507f91f into 6e9bc4bd75bd4ebfbb6e9f8e058d39c8(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:37:06,193 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:06,193 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., storeName=5165da7af32dbdfd119fd1fa7507f91f/C, priority=13, startTime=1730972225983; duration=0sec 2024-11-07T09:37:06,193 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:06,193 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5165da7af32dbdfd119fd1fa7507f91f:C 2024-11-07T09:37:06,193 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=26.84 KB/27480 for 5165da7af32dbdfd119fd1fa7507f91f in 168ms, sequenceid=73, compaction requested=false 2024-11-07T09:37:06,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:06,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:06,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-07T09:37:06,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-07T09:37:06,198 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-07T09:37:06,198 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 789 msec 2024-11-07T09:37:06,200 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 795 msec 2024-11-07T09:37:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 5165da7af32dbdfd119fd1fa7507f91f 2024-11-07T09:37:06,208 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:37:06,208 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5165da7af32dbdfd119fd1fa7507f91f 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-07T09:37:06,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/45d6fc8e43944071a62278cd0a1e7ebc is 50, key is test_row_0/A:col10/1730972226205/Put/seqid=0 2024-11-07T09:37:06,229 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T09:37:06,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742096_1272 (size=12001) 2024-11-07T09:37:06,243 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/45d6fc8e43944071a62278cd0a1e7ebc 2024-11-07T09:37:06,264 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/18d5dc17b3d44bc8940944c6f721f0af is 50, key is test_row_0/B:col10/1730972226205/Put/seqid=0 2024-11-07T09:37:06,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:06,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42842 deadline: 1730972286265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:06,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:06,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:06,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:06,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:06,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730972286272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:06,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:06,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:06,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:06,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:06,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972286273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:06,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:06,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:06,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:06,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:06,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42826 deadline: 1730972286273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:06,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:06,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:06,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:06,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:06,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1730972286273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:06,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:06,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:06,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:06,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742097_1273 (size=12001) 2024-11-07T09:37:06,287 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/18d5dc17b3d44bc8940944c6f721f0af 2024-11-07T09:37:06,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/175bdae664884b6baa57df81c0bbd195 is 50, key is test_row_0/C:col10/1730972226205/Put/seqid=0 2024-11-07T09:37:06,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742098_1274 (size=12001) 2024-11-07T09:37:06,327 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/175bdae664884b6baa57df81c0bbd195 2024-11-07T09:37:06,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/45d6fc8e43944071a62278cd0a1e7ebc as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/45d6fc8e43944071a62278cd0a1e7ebc 2024-11-07T09:37:06,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/45d6fc8e43944071a62278cd0a1e7ebc, entries=150, sequenceid=88, filesize=11.7 K 2024-11-07T09:37:06,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/18d5dc17b3d44bc8940944c6f721f0af as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/18d5dc17b3d44bc8940944c6f721f0af 2024-11-07T09:37:06,355 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/18d5dc17b3d44bc8940944c6f721f0af, entries=150, sequenceid=88, filesize=11.7 K 2024-11-07T09:37:06,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/175bdae664884b6baa57df81c0bbd195 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/175bdae664884b6baa57df81c0bbd195 2024-11-07T09:37:06,364 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/175bdae664884b6baa57df81c0bbd195, entries=150, sequenceid=88, filesize=11.7 K 2024-11-07T09:37:06,365 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 5165da7af32dbdfd119fd1fa7507f91f in 157ms, sequenceid=88, compaction requested=true 2024-11-07T09:37:06,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:06,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5165da7af32dbdfd119fd1fa7507f91f:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T09:37:06,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:06,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5165da7af32dbdfd119fd1fa7507f91f:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T09:37:06,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:37:06,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5165da7af32dbdfd119fd1fa7507f91f:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:37:06,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T09:37:06,365 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 3 compacting, 2 eligible, 16 blocking 2024-11-07T09:37:06,366 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-07T09:37:06,366 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-07T09:37:06,366 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. because compaction request was cancelled 2024-11-07T09:37:06,366 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5165da7af32dbdfd119fd1fa7507f91f:A 2024-11-07T09:37:06,366 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:37:06,367 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:37:06,368 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 5165da7af32dbdfd119fd1fa7507f91f/B is initiating minor compaction (all files) 2024-11-07T09:37:06,368 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5165da7af32dbdfd119fd1fa7507f91f/B in TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:06,368 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/f5a7a611fe294bc49cba015cfc0c3ed3, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/b72c10544cb04b8b9a4263601a34a73a, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/18d5dc17b3d44bc8940944c6f721f0af] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp, totalSize=35.3 K 2024-11-07T09:37:06,368 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting f5a7a611fe294bc49cba015cfc0c3ed3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1730972225474 2024-11-07T09:37:06,369 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting b72c10544cb04b8b9a4263601a34a73a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1730972225507 2024-11-07T09:37:06,369 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 18d5dc17b3d44bc8940944c6f721f0af, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1730972226205 2024-11-07T09:37:06,388 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5165da7af32dbdfd119fd1fa7507f91f#B#compaction#228 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:37:06,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 5165da7af32dbdfd119fd1fa7507f91f 2024-11-07T09:37:06,389 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:37:06,389 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5165da7af32dbdfd119fd1fa7507f91f 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T09:37:06,389 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/d1350934596941f7bf5a8aeb25184c17 is 50, key is test_row_0/B:col10/1730972226205/Put/seqid=0 2024-11-07T09:37:06,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/30c9caef9cbe416fb69d974b947c95aa is 50, key is test_row_0/A:col10/1730972226269/Put/seqid=0 2024-11-07T09:37:06,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:06,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42842 deadline: 1730972286400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:06,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:06,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:06,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:06,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:06,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972286402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:06,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:06,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:06,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:06,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742100_1276 (size=12001) 2024-11-07T09:37:06,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:06,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730972286415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:06,432 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:06,432 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:06,432 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:06,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:06,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42826 deadline: 1730972286428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:06,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:06,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1730972286429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:06,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:06,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:06,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:06,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:06,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:06,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:06,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742099_1275 (size=12207) 2024-11-07T09:37:06,440 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/30c9caef9cbe416fb69d974b947c95aa 2024-11-07T09:37:06,466 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/f8351f23305f4d8ab65ecefe616ff687 is 50, key is test_row_0/B:col10/1730972226269/Put/seqid=0 2024-11-07T09:37:06,485 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/4aa193391d4a4cbe9d33aba3559a39f7 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/4aa193391d4a4cbe9d33aba3559a39f7 2024-11-07T09:37:06,491 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5165da7af32dbdfd119fd1fa7507f91f/A of 5165da7af32dbdfd119fd1fa7507f91f into 4aa193391d4a4cbe9d33aba3559a39f7(size=11.8 K), total size for store is 35.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:37:06,491 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:06,491 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., storeName=5165da7af32dbdfd119fd1fa7507f91f/A, priority=13, startTime=1730972225983; duration=0sec 2024-11-07T09:37:06,491 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:37:06,491 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5165da7af32dbdfd119fd1fa7507f91f:A 2024-11-07T09:37:06,491 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:37:06,492 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:37:06,492 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 5165da7af32dbdfd119fd1fa7507f91f/C is initiating minor compaction (all files) 2024-11-07T09:37:06,492 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5165da7af32dbdfd119fd1fa7507f91f/C in TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:06,492 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/6e9bc4bd75bd4ebfbb6e9f8e058d39c8, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/bb1982c8ac434b918cfb128749d9c913, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/175bdae664884b6baa57df81c0bbd195] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp, totalSize=35.3 K 2024-11-07T09:37:06,493 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e9bc4bd75bd4ebfbb6e9f8e058d39c8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1730972225474 2024-11-07T09:37:06,494 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb1982c8ac434b918cfb128749d9c913, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1730972225507 2024-11-07T09:37:06,494 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 175bdae664884b6baa57df81c0bbd195, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1730972226205 2024-11-07T09:37:06,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742101_1277 (size=12001) 2024-11-07T09:37:06,514 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5165da7af32dbdfd119fd1fa7507f91f#C#compaction#231 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:37:06,515 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/e3d86abb5edd4bf98bfb2cfcc8609a8d is 50, key is test_row_0/C:col10/1730972226205/Put/seqid=0 2024-11-07T09:37:06,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:06,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972286533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:06,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-07T09:37:06,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:06,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:06,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:06,543 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:37:06,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:06,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42842 deadline: 1730972286533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:06,543 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T09:37:06,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:06,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:06,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:06,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-07T09:37:06,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-07T09:37:06,546 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T09:37:06,547 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T09:37:06,547 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T09:37:06,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:06,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730972286543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:06,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:06,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:06,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:06,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:06,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:06,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42826 deadline: 1730972286543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:06,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:06,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:06,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:06,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1730972286543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:06,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:06,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:06,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:06,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742102_1278 (size=12207) 2024-11-07T09:37:06,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-07T09:37:06,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-07T09:37:06,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:06,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. as already flushing 2024-11-07T09:37:06,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:06,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:06,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:06,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:06,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:06,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:06,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972286753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:06,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42842 deadline: 1730972286753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:06,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:06,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:06,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:06,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:06,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:06,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:06,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:06,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730972286762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:06,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:06,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42826 deadline: 1730972286763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:06,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:06,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1730972286763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:06,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:06,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:06,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:06,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:06,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:06,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:06,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:06,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:06,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:06,848 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/d1350934596941f7bf5a8aeb25184c17 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/d1350934596941f7bf5a8aeb25184c17 2024-11-07T09:37:06,854 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5165da7af32dbdfd119fd1fa7507f91f/B of 5165da7af32dbdfd119fd1fa7507f91f into d1350934596941f7bf5a8aeb25184c17(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:37:06,854 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:06,854 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., storeName=5165da7af32dbdfd119fd1fa7507f91f/B, priority=13, startTime=1730972226365; duration=0sec 2024-11-07T09:37:06,854 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:06,854 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5165da7af32dbdfd119fd1fa7507f91f:B 2024-11-07T09:37:06,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-07T09:37:06,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:06,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. as already flushing 2024-11-07T09:37:06,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:06,856 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:06,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:06,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:06,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-07T09:37:06,907 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/f8351f23305f4d8ab65ecefe616ff687 2024-11-07T09:37:06,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/3edf0a5a2c074ba59311f7c41f312b37 is 50, key is test_row_0/C:col10/1730972226269/Put/seqid=0 2024-11-07T09:37:06,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742103_1279 (size=12001) 2024-11-07T09:37:06,933 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/3edf0a5a2c074ba59311f7c41f312b37 2024-11-07T09:37:06,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/30c9caef9cbe416fb69d974b947c95aa as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/30c9caef9cbe416fb69d974b947c95aa 2024-11-07T09:37:06,943 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/30c9caef9cbe416fb69d974b947c95aa, entries=150, sequenceid=114, filesize=11.7 K 2024-11-07T09:37:06,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/f8351f23305f4d8ab65ecefe616ff687 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/f8351f23305f4d8ab65ecefe616ff687 2024-11-07T09:37:06,948 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/f8351f23305f4d8ab65ecefe616ff687, entries=150, sequenceid=114, filesize=11.7 K 2024-11-07T09:37:06,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/3edf0a5a2c074ba59311f7c41f312b37 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/3edf0a5a2c074ba59311f7c41f312b37 2024-11-07T09:37:06,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/3edf0a5a2c074ba59311f7c41f312b37, entries=150, sequenceid=114, filesize=11.7 K 2024-11-07T09:37:06,954 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 5165da7af32dbdfd119fd1fa7507f91f in 564ms, sequenceid=114, compaction requested=true 2024-11-07T09:37:06,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:06,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5165da7af32dbdfd119fd1fa7507f91f:A, priority=-2147483648, current under compaction store size is 2 2024-11-07T09:37:06,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:06,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5165da7af32dbdfd119fd1fa7507f91f:B, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:37:06,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:37:06,954 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T09:37:06,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5165da7af32dbdfd119fd1fa7507f91f:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:37:06,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T09:37:06,956 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T09:37:06,956 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 5165da7af32dbdfd119fd1fa7507f91f/A is initiating minor compaction (all files) 2024-11-07T09:37:06,956 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5165da7af32dbdfd119fd1fa7507f91f/A in TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:06,956 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/4aa193391d4a4cbe9d33aba3559a39f7, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/f6903116b51345c8babd0eada216f02d, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/45d6fc8e43944071a62278cd0a1e7ebc, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/30c9caef9cbe416fb69d974b947c95aa] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp, totalSize=47.0 K 2024-11-07T09:37:06,957 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 4aa193391d4a4cbe9d33aba3559a39f7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1730972225474 2024-11-07T09:37:06,958 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting f6903116b51345c8babd0eada216f02d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1730972225507 2024-11-07T09:37:06,958 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 45d6fc8e43944071a62278cd0a1e7ebc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1730972226205 2024-11-07T09:37:06,959 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 30c9caef9cbe416fb69d974b947c95aa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1730972226269 2024-11-07T09:37:06,991 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/e3d86abb5edd4bf98bfb2cfcc8609a8d as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/e3d86abb5edd4bf98bfb2cfcc8609a8d 2024-11-07T09:37:06,996 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5165da7af32dbdfd119fd1fa7507f91f#A#compaction#233 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:37:06,997 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/d75bb0c439da419ebf5558fedbc1e76d is 50, key is test_row_0/A:col10/1730972226269/Put/seqid=0 2024-11-07T09:37:06,998 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5165da7af32dbdfd119fd1fa7507f91f/C of 5165da7af32dbdfd119fd1fa7507f91f into e3d86abb5edd4bf98bfb2cfcc8609a8d(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:37:06,998 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:06,998 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., storeName=5165da7af32dbdfd119fd1fa7507f91f/C, priority=13, startTime=1730972226365; duration=0sec 2024-11-07T09:37:06,998 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T09:37:06,998 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5165da7af32dbdfd119fd1fa7507f91f:C 2024-11-07T09:37:06,998 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5165da7af32dbdfd119fd1fa7507f91f:C 2024-11-07T09:37:06,998 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-07T09:37:06,999 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-07T09:37:06,999 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-07T09:37:07,000 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. because compaction request was cancelled 2024-11-07T09:37:07,000 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5165da7af32dbdfd119fd1fa7507f91f:B 2024-11-07T09:37:07,001 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-07T09:37:07,002 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-07T09:37:07,002 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-07T09:37:07,002 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. because compaction request was cancelled 2024-11-07T09:37:07,002 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5165da7af32dbdfd119fd1fa7507f91f:C 2024-11-07T09:37:07,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742104_1280 (size=12241) 2024-11-07T09:37:07,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-07T09:37:07,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:07,008 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 5165da7af32dbdfd119fd1fa7507f91f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T09:37:07,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/78c512af0fe54c6cb5a3115ab9f8bd1e is 50, key is test_row_0/A:col10/1730972226406/Put/seqid=0 2024-11-07T09:37:07,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742105_1281 (size=12001) 2024-11-07T09:37:07,031 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/78c512af0fe54c6cb5a3115ab9f8bd1e 2024-11-07T09:37:07,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/ac7998e7d8b144dda5f3b41b4d664751 is 50, key is test_row_0/B:col10/1730972226406/Put/seqid=0 2024-11-07T09:37:07,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 5165da7af32dbdfd119fd1fa7507f91f 2024-11-07T09:37:07,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. as already flushing 2024-11-07T09:37:07,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742106_1282 (size=12001) 2024-11-07T09:37:07,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:07,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42842 deadline: 1730972287099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:07,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:07,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:07,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:07,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:07,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1730972287100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:07,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:07,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972287102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:07,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:07,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:07,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:07,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:07,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730972287103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:07,108 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:07,108 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:07,108 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:07,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:07,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:07,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:07,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:07,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42826 deadline: 1730972287105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:07,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:07,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:07,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:07,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-07T09:37:07,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:07,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42842 deadline: 1730972287213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:07,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:07,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:07,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:07,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:07,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730972287213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:07,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:07,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:07,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:07,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:07,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972287213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:07,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:07,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:07,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:07,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:07,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1730972287214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:07,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:07,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:07,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:07,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:07,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42826 deadline: 1730972287214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:07,219 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:07,219 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:07,219 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:07,411 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/d75bb0c439da419ebf5558fedbc1e76d as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/d75bb0c439da419ebf5558fedbc1e76d 2024-11-07T09:37:07,417 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5165da7af32dbdfd119fd1fa7507f91f/A of 5165da7af32dbdfd119fd1fa7507f91f into d75bb0c439da419ebf5558fedbc1e76d(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:37:07,417 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:07,417 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., storeName=5165da7af32dbdfd119fd1fa7507f91f/A, priority=12, startTime=1730972226954; duration=0sec 2024-11-07T09:37:07,418 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:07,418 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5165da7af32dbdfd119fd1fa7507f91f:A 2024-11-07T09:37:07,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:07,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42842 deadline: 1730972287423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:07,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:07,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:07,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:07,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:07,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972287423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:07,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:07,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:07,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:07,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:07,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730972287423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:07,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:07,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:07,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:07,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:07,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1730972287423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:07,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:07,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42826 deadline: 1730972287423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:07,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:07,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:07,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:07,428 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:07,428 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:07,428 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:07,475 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/ac7998e7d8b144dda5f3b41b4d664751 2024-11-07T09:37:07,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/76a80e9d95bc48ab903417fd5fba44c8 is 50, key is test_row_0/C:col10/1730972226406/Put/seqid=0 2024-11-07T09:37:07,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742107_1283 (size=12001) 2024-11-07T09:37:07,503 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/76a80e9d95bc48ab903417fd5fba44c8 2024-11-07T09:37:07,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/78c512af0fe54c6cb5a3115ab9f8bd1e as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/78c512af0fe54c6cb5a3115ab9f8bd1e 2024-11-07T09:37:07,514 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/78c512af0fe54c6cb5a3115ab9f8bd1e, entries=150, sequenceid=128, filesize=11.7 K 2024-11-07T09:37:07,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/ac7998e7d8b144dda5f3b41b4d664751 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/ac7998e7d8b144dda5f3b41b4d664751 2024-11-07T09:37:07,525 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/ac7998e7d8b144dda5f3b41b4d664751, entries=150, sequenceid=128, filesize=11.7 K 2024-11-07T09:37:07,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/76a80e9d95bc48ab903417fd5fba44c8 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/76a80e9d95bc48ab903417fd5fba44c8 2024-11-07T09:37:07,535 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/76a80e9d95bc48ab903417fd5fba44c8, entries=150, sequenceid=128, filesize=11.7 K 2024-11-07T09:37:07,536 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 5165da7af32dbdfd119fd1fa7507f91f in 528ms, sequenceid=128, compaction requested=true 2024-11-07T09:37:07,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:07,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:07,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-07T09:37:07,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-07T09:37:07,540 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-07T09:37:07,540 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 991 msec 2024-11-07T09:37:07,542 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 998 msec 2024-11-07T09:37:07,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-07T09:37:07,683 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:37:07,684 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T09:37:07,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-07T09:37:07,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-07T09:37:07,686 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T09:37:07,687 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T09:37:07,687 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T09:37:07,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 5165da7af32dbdfd119fd1fa7507f91f 2024-11-07T09:37:07,736 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:37:07,736 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5165da7af32dbdfd119fd1fa7507f91f 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-07T09:37:07,744 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/ed34b001e7914b6cb45dcb9627954b1b is 50, key is test_row_0/A:col10/1730972227734/Put/seqid=0 2024-11-07T09:37:07,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:07,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1730972287740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:07,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:07,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42826 deadline: 1730972287742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:07,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:07,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730972287743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:07,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:07,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:07,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:07,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:07,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:07,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:07,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:07,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:07,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:07,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742108_1284 (size=12151) 2024-11-07T09:37:07,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:07,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42842 deadline: 1730972287747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:07,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:07,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972287748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:07,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:07,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:07,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:07,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:07,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:07,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:07,756 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/ed34b001e7914b6cb45dcb9627954b1b 2024-11-07T09:37:07,768 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/36b10cd6f8bf44e590959ae841183aec is 50, key is test_row_0/B:col10/1730972227734/Put/seqid=0 2024-11-07T09:37:07,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742109_1285 (size=12151) 2024-11-07T09:37:07,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-07T09:37:07,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-07T09:37:07,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:07,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. as already flushing 2024-11-07T09:37:07,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:07,840 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:07,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:07,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:07,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:07,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730972287853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:07,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:07,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:07,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:07,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:07,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42826 deadline: 1730972287853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:07,856 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:07,856 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:07,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:07,856 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:07,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1730972287853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:07,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:07,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:07,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:07,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:07,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42842 deadline: 1730972287863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:07,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:07,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:07,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:07,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:07,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972287863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:07,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:07,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:07,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:07,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-07T09:37:07,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:07,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. as already flushing 2024-11-07T09:37:07,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:07,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:07,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:07,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:08,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-07T09:37:08,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:08,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730972288063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:08,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:08,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:08,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:08,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:08,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42826 deadline: 1730972288063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:08,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:08,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:08,069 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:08,069 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:08,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1730972288064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:08,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:08,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:08,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:08,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:08,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42842 deadline: 1730972288073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:08,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:08,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:08,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972288073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:08,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:08,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:08,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:08,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:08,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:08,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-07T09:37:08,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:08,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. as already flushing 2024-11-07T09:37:08,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:08,147 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:08,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:08,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:08,173 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/36b10cd6f8bf44e590959ae841183aec 2024-11-07T09:37:08,187 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/bfc4e336017c4ce7a18a1ebf73784544 is 50, key is test_row_0/C:col10/1730972227734/Put/seqid=0 2024-11-07T09:37:08,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742110_1286 (size=12151) 2024-11-07T09:37:08,194 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/bfc4e336017c4ce7a18a1ebf73784544 2024-11-07T09:37:08,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/ed34b001e7914b6cb45dcb9627954b1b as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/ed34b001e7914b6cb45dcb9627954b1b 2024-11-07T09:37:08,212 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/ed34b001e7914b6cb45dcb9627954b1b, entries=150, sequenceid=156, filesize=11.9 K 2024-11-07T09:37:08,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/36b10cd6f8bf44e590959ae841183aec as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/36b10cd6f8bf44e590959ae841183aec 2024-11-07T09:37:08,218 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/36b10cd6f8bf44e590959ae841183aec, entries=150, sequenceid=156, filesize=11.9 K 2024-11-07T09:37:08,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/bfc4e336017c4ce7a18a1ebf73784544 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/bfc4e336017c4ce7a18a1ebf73784544 2024-11-07T09:37:08,223 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/bfc4e336017c4ce7a18a1ebf73784544, entries=150, sequenceid=156, filesize=11.9 K 2024-11-07T09:37:08,224 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 5165da7af32dbdfd119fd1fa7507f91f in 488ms, sequenceid=156, compaction requested=true 2024-11-07T09:37:08,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:08,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5165da7af32dbdfd119fd1fa7507f91f:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T09:37:08,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:08,224 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:37:08,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5165da7af32dbdfd119fd1fa7507f91f:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T09:37:08,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:08,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5165da7af32dbdfd119fd1fa7507f91f:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:37:08,224 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T09:37:08,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:37:08,225 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:37:08,225 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 5165da7af32dbdfd119fd1fa7507f91f/A is initiating minor compaction (all files) 2024-11-07T09:37:08,225 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5165da7af32dbdfd119fd1fa7507f91f/A in TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:08,225 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T09:37:08,225 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/d75bb0c439da419ebf5558fedbc1e76d, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/78c512af0fe54c6cb5a3115ab9f8bd1e, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/ed34b001e7914b6cb45dcb9627954b1b] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp, totalSize=35.5 K 2024-11-07T09:37:08,226 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 5165da7af32dbdfd119fd1fa7507f91f/B is initiating minor compaction (all files) 2024-11-07T09:37:08,226 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5165da7af32dbdfd119fd1fa7507f91f/B in TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:08,226 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/d1350934596941f7bf5a8aeb25184c17, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/f8351f23305f4d8ab65ecefe616ff687, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/ac7998e7d8b144dda5f3b41b4d664751, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/36b10cd6f8bf44e590959ae841183aec] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp, totalSize=47.2 K 2024-11-07T09:37:08,226 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting d75bb0c439da419ebf5558fedbc1e76d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1730972226269 2024-11-07T09:37:08,226 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting d1350934596941f7bf5a8aeb25184c17, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1730972226205 2024-11-07T09:37:08,227 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78c512af0fe54c6cb5a3115ab9f8bd1e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1730972226398 2024-11-07T09:37:08,227 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting f8351f23305f4d8ab65ecefe616ff687, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1730972226269 2024-11-07T09:37:08,227 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed34b001e7914b6cb45dcb9627954b1b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1730972227103 2024-11-07T09:37:08,227 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting ac7998e7d8b144dda5f3b41b4d664751, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1730972226398 2024-11-07T09:37:08,228 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 36b10cd6f8bf44e590959ae841183aec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1730972227103 2024-11-07T09:37:08,240 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5165da7af32dbdfd119fd1fa7507f91f#A#compaction#240 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:37:08,242 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/157c6d58e9d8403ca2bf402c42f97646 is 50, key is test_row_0/A:col10/1730972227734/Put/seqid=0 2024-11-07T09:37:08,245 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5165da7af32dbdfd119fd1fa7507f91f#B#compaction#241 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:37:08,246 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/5e7709e8e0c34a11a55b40643682f34a is 50, key is test_row_0/B:col10/1730972227734/Put/seqid=0 2024-11-07T09:37:08,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742111_1287 (size=12493) 2024-11-07T09:37:08,281 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/157c6d58e9d8403ca2bf402c42f97646 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/157c6d58e9d8403ca2bf402c42f97646 2024-11-07T09:37:08,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742112_1288 (size=12493) 2024-11-07T09:37:08,290 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5165da7af32dbdfd119fd1fa7507f91f/A of 5165da7af32dbdfd119fd1fa7507f91f into 157c6d58e9d8403ca2bf402c42f97646(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:37:08,290 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:08,290 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., storeName=5165da7af32dbdfd119fd1fa7507f91f/A, priority=13, startTime=1730972228224; duration=0sec 2024-11-07T09:37:08,290 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:37:08,290 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5165da7af32dbdfd119fd1fa7507f91f:A 2024-11-07T09:37:08,290 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T09:37:08,291 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T09:37:08,291 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 5165da7af32dbdfd119fd1fa7507f91f/C is initiating minor compaction (all files) 2024-11-07T09:37:08,291 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5165da7af32dbdfd119fd1fa7507f91f/C in TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:08,292 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/e3d86abb5edd4bf98bfb2cfcc8609a8d, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/3edf0a5a2c074ba59311f7c41f312b37, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/76a80e9d95bc48ab903417fd5fba44c8, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/bfc4e336017c4ce7a18a1ebf73784544] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp, totalSize=47.2 K 2024-11-07T09:37:08,292 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3d86abb5edd4bf98bfb2cfcc8609a8d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1730972226205 2024-11-07T09:37:08,292 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3edf0a5a2c074ba59311f7c41f312b37, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1730972226269 2024-11-07T09:37:08,293 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76a80e9d95bc48ab903417fd5fba44c8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1730972226398 2024-11-07T09:37:08,293 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfc4e336017c4ce7a18a1ebf73784544, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1730972227103 2024-11-07T09:37:08,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-07T09:37:08,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:08,300 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 5165da7af32dbdfd119fd1fa7507f91f 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-07T09:37:08,305 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5165da7af32dbdfd119fd1fa7507f91f#C#compaction#242 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:37:08,305 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/553e92525a014504b227e284d6b795b9 is 50, key is test_row_0/C:col10/1730972227734/Put/seqid=0 2024-11-07T09:37:08,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/7ee99ea076304c13accbfced55cf7ab0 is 50, key is test_row_0/A:col10/1730972227740/Put/seqid=0 2024-11-07T09:37:08,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-07T09:37:08,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742113_1289 (size=12493) 2024-11-07T09:37:08,319 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/553e92525a014504b227e284d6b795b9 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/553e92525a014504b227e284d6b795b9 2024-11-07T09:37:08,324 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5165da7af32dbdfd119fd1fa7507f91f/C of 5165da7af32dbdfd119fd1fa7507f91f into 553e92525a014504b227e284d6b795b9(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:37:08,324 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:08,325 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., storeName=5165da7af32dbdfd119fd1fa7507f91f/C, priority=12, startTime=1730972228224; duration=0sec 2024-11-07T09:37:08,325 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:08,325 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5165da7af32dbdfd119fd1fa7507f91f:C 2024-11-07T09:37:08,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742114_1290 (size=12151) 2024-11-07T09:37:08,352 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/7ee99ea076304c13accbfced55cf7ab0 2024-11-07T09:37:08,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/a21cf5509f6440f285d2462a6c2b554c is 50, key is test_row_0/B:col10/1730972227740/Put/seqid=0 2024-11-07T09:37:08,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742115_1291 (size=12151) 2024-11-07T09:37:08,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 5165da7af32dbdfd119fd1fa7507f91f 2024-11-07T09:37:08,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. as already flushing 2024-11-07T09:37:08,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:08,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42826 deadline: 1730972288407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:08,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:08,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:08,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:08,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:08,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42842 deadline: 1730972288407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:08,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:08,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:08,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:08,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:08,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972288411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:08,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:08,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:08,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:08,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:08,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730972288412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:08,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:08,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:08,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:08,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1730972288412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:08,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:08,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:08,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:08,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:08,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:08,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42842 deadline: 1730972288523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:08,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:08,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42826 deadline: 1730972288523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:08,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:08,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:08,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:08,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:08,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:08,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:08,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972288523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:08,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:08,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:08,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:08,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:08,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:08,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730972288523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:08,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1730972288524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:08,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:08,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:08,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:08,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:08,530 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:08,530 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:08,530 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:08,697 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/5e7709e8e0c34a11a55b40643682f34a as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/5e7709e8e0c34a11a55b40643682f34a 2024-11-07T09:37:08,703 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5165da7af32dbdfd119fd1fa7507f91f/B of 5165da7af32dbdfd119fd1fa7507f91f into 5e7709e8e0c34a11a55b40643682f34a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:37:08,703 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:08,703 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., storeName=5165da7af32dbdfd119fd1fa7507f91f/B, priority=12, startTime=1730972228224; duration=0sec 2024-11-07T09:37:08,703 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:08,703 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5165da7af32dbdfd119fd1fa7507f91f:B 2024-11-07T09:37:08,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:08,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42842 deadline: 1730972288733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:08,737 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:08,737 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:08,737 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:08,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:08,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42826 deadline: 1730972288733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:08,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:08,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:08,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:08,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:08,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972288733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:08,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:08,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:08,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:08,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:08,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1730972288733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:08,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:08,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:08,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:08,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:08,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730972288733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:08,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:08,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:08,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:08,771 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/a21cf5509f6440f285d2462a6c2b554c 2024-11-07T09:37:08,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/5f5f0c839ef34f80bb281b6fd80bec40 is 50, key is test_row_0/C:col10/1730972227740/Put/seqid=0 2024-11-07T09:37:08,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742116_1292 (size=12151) 2024-11-07T09:37:08,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-07T09:37:09,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:09,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42842 deadline: 1730972289043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:09,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:09,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:09,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:09,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:09,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42826 deadline: 1730972289043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:09,047 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:09,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:09,047 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:09,047 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:09,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972289043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:09,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:09,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:09,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:09,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:09,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1730972289043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:09,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:09,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730972289043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:09,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:09,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:09,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:09,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:09,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:09,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:09,186 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/5f5f0c839ef34f80bb281b6fd80bec40 2024-11-07T09:37:09,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/7ee99ea076304c13accbfced55cf7ab0 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/7ee99ea076304c13accbfced55cf7ab0 2024-11-07T09:37:09,196 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/7ee99ea076304c13accbfced55cf7ab0, entries=150, sequenceid=166, filesize=11.9 K 2024-11-07T09:37:09,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/a21cf5509f6440f285d2462a6c2b554c as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/a21cf5509f6440f285d2462a6c2b554c 2024-11-07T09:37:09,202 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/a21cf5509f6440f285d2462a6c2b554c, entries=150, sequenceid=166, filesize=11.9 K 2024-11-07T09:37:09,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/5f5f0c839ef34f80bb281b6fd80bec40 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/5f5f0c839ef34f80bb281b6fd80bec40 2024-11-07T09:37:09,209 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/5f5f0c839ef34f80bb281b6fd80bec40, entries=150, sequenceid=166, filesize=11.9 K 2024-11-07T09:37:09,210 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 5165da7af32dbdfd119fd1fa7507f91f in 910ms, sequenceid=166, compaction requested=false 2024-11-07T09:37:09,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:09,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:09,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-07T09:37:09,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-07T09:37:09,215 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-07T09:37:09,215 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5250 sec 2024-11-07T09:37:09,216 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.5310 sec 2024-11-07T09:37:09,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 5165da7af32dbdfd119fd1fa7507f91f 2024-11-07T09:37:09,556 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:37:09,556 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5165da7af32dbdfd119fd1fa7507f91f 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-07T09:37:09,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:09,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972289563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:09,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:09,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:09,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:09,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:09,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42842 deadline: 1730972289565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:09,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:09,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:09,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:09,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/468523535622432f9b9a4760d669b7df is 50, key is test_row_0/A:col10/1730972229556/Put/seqid=0 2024-11-07T09:37:09,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:09,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42826 deadline: 1730972289571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:09,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:09,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1730972289571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:09,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:09,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730972289572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:09,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:09,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:09,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:09,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:09,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:09,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:09,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:09,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:09,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:09,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742117_1293 (size=12151) 2024-11-07T09:37:09,615 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/468523535622432f9b9a4760d669b7df 2024-11-07T09:37:09,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/120008ccfe594daea80078a90b4d6e1a is 50, key is test_row_0/B:col10/1730972229556/Put/seqid=0 2024-11-07T09:37:09,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742118_1294 (size=12151) 2024-11-07T09:37:09,635 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/120008ccfe594daea80078a90b4d6e1a 2024-11-07T09:37:09,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/1f9d5d0470c049d69b09a9391e4bda19 is 50, key is test_row_0/C:col10/1730972229556/Put/seqid=0 2024-11-07T09:37:09,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742119_1295 (size=12151) 2024-11-07T09:37:09,662 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/1f9d5d0470c049d69b09a9391e4bda19 2024-11-07T09:37:09,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/468523535622432f9b9a4760d669b7df as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/468523535622432f9b9a4760d669b7df 2024-11-07T09:37:09,682 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/468523535622432f9b9a4760d669b7df, entries=150, sequenceid=198, filesize=11.9 K 2024-11-07T09:37:09,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/120008ccfe594daea80078a90b4d6e1a as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/120008ccfe594daea80078a90b4d6e1a 2024-11-07T09:37:09,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:09,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1730972289683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:09,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:09,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:09,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:09,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:09,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42826 deadline: 1730972289683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:09,686 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:09,686 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:09,686 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:09,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:09,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730972289683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:09,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:09,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:09,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:09,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:09,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42842 deadline: 1730972289683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:09,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:09,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:09,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:09,689 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/120008ccfe594daea80078a90b4d6e1a, entries=150, sequenceid=198, filesize=11.9 K 2024-11-07T09:37:09,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/1f9d5d0470c049d69b09a9391e4bda19 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/1f9d5d0470c049d69b09a9391e4bda19 2024-11-07T09:37:09,697 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/1f9d5d0470c049d69b09a9391e4bda19, entries=150, sequenceid=198, filesize=11.9 K 2024-11-07T09:37:09,698 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=20.13 KB/20610 for 5165da7af32dbdfd119fd1fa7507f91f in 142ms, sequenceid=198, compaction requested=true 2024-11-07T09:37:09,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:09,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5165da7af32dbdfd119fd1fa7507f91f:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T09:37:09,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:09,698 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:37:09,698 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:37:09,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5165da7af32dbdfd119fd1fa7507f91f:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T09:37:09,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:09,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5165da7af32dbdfd119fd1fa7507f91f:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:37:09,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:37:09,701 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:37:09,701 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 5165da7af32dbdfd119fd1fa7507f91f/A is initiating minor compaction (all files) 2024-11-07T09:37:09,701 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:37:09,701 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 5165da7af32dbdfd119fd1fa7507f91f/B is initiating minor compaction (all files) 2024-11-07T09:37:09,701 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5165da7af32dbdfd119fd1fa7507f91f/A in TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:09,701 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5165da7af32dbdfd119fd1fa7507f91f/B in TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:09,701 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/157c6d58e9d8403ca2bf402c42f97646, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/7ee99ea076304c13accbfced55cf7ab0, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/468523535622432f9b9a4760d669b7df] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp, totalSize=35.9 K 2024-11-07T09:37:09,701 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/5e7709e8e0c34a11a55b40643682f34a, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/a21cf5509f6440f285d2462a6c2b554c, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/120008ccfe594daea80078a90b4d6e1a] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp, totalSize=35.9 K 2024-11-07T09:37:09,702 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e7709e8e0c34a11a55b40643682f34a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1730972227103 2024-11-07T09:37:09,702 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 157c6d58e9d8403ca2bf402c42f97646, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1730972227103 2024-11-07T09:37:09,702 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting a21cf5509f6440f285d2462a6c2b554c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1730972227739 2024-11-07T09:37:09,703 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ee99ea076304c13accbfced55cf7ab0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1730972227739 2024-11-07T09:37:09,704 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 120008ccfe594daea80078a90b4d6e1a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1730972229554 2024-11-07T09:37:09,705 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 468523535622432f9b9a4760d669b7df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1730972229554 2024-11-07T09:37:09,722 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5165da7af32dbdfd119fd1fa7507f91f#B#compaction#249 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:37:09,723 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/9c93a880b0464030823dd86c8215b165 is 50, key is test_row_0/B:col10/1730972229556/Put/seqid=0 2024-11-07T09:37:09,735 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5165da7af32dbdfd119fd1fa7507f91f#A#compaction#250 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:37:09,735 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/fdf4034adfa2457e8df7e60dae72f0d2 is 50, key is test_row_0/A:col10/1730972229556/Put/seqid=0 2024-11-07T09:37:09,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742120_1296 (size=12595) 2024-11-07T09:37:09,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742121_1297 (size=12595) 2024-11-07T09:37:09,791 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/fdf4034adfa2457e8df7e60dae72f0d2 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/fdf4034adfa2457e8df7e60dae72f0d2 2024-11-07T09:37:09,799 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5165da7af32dbdfd119fd1fa7507f91f/A of 5165da7af32dbdfd119fd1fa7507f91f into fdf4034adfa2457e8df7e60dae72f0d2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:37:09,799 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:09,799 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., storeName=5165da7af32dbdfd119fd1fa7507f91f/A, priority=13, startTime=1730972229698; duration=0sec 2024-11-07T09:37:09,799 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:37:09,799 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5165da7af32dbdfd119fd1fa7507f91f:A 2024-11-07T09:37:09,799 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:37:09,802 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:37:09,802 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 5165da7af32dbdfd119fd1fa7507f91f/C is initiating minor compaction (all files) 2024-11-07T09:37:09,802 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5165da7af32dbdfd119fd1fa7507f91f/C in TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:09,802 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/553e92525a014504b227e284d6b795b9, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/5f5f0c839ef34f80bb281b6fd80bec40, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/1f9d5d0470c049d69b09a9391e4bda19] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp, totalSize=35.9 K 2024-11-07T09:37:09,803 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 553e92525a014504b227e284d6b795b9, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1730972227103 2024-11-07T09:37:09,803 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f5f0c839ef34f80bb281b6fd80bec40, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1730972227739 2024-11-07T09:37:09,803 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f9d5d0470c049d69b09a9391e4bda19, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1730972229554 2024-11-07T09:37:09,816 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5165da7af32dbdfd119fd1fa7507f91f#C#compaction#251 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:37:09,817 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/aab565688d664ead8cfdb44eb9d6a626 is 50, key is test_row_0/C:col10/1730972229556/Put/seqid=0 2024-11-07T09:37:09,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-07T09:37:09,833 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:37:09,834 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T09:37:09,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-07T09:37:09,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-07T09:37:09,837 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T09:37:09,838 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T09:37:09,838 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T09:37:09,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742122_1298 (size=12595) 2024-11-07T09:37:09,851 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/aab565688d664ead8cfdb44eb9d6a626 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/aab565688d664ead8cfdb44eb9d6a626 2024-11-07T09:37:09,858 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5165da7af32dbdfd119fd1fa7507f91f/C of 5165da7af32dbdfd119fd1fa7507f91f into aab565688d664ead8cfdb44eb9d6a626(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:37:09,858 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:09,858 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., storeName=5165da7af32dbdfd119fd1fa7507f91f/C, priority=13, startTime=1730972229699; duration=0sec 2024-11-07T09:37:09,858 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:09,858 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5165da7af32dbdfd119fd1fa7507f91f:C 2024-11-07T09:37:09,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 5165da7af32dbdfd119fd1fa7507f91f 2024-11-07T09:37:09,907 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:37:09,907 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5165da7af32dbdfd119fd1fa7507f91f 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-07T09:37:09,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/eda6886d0f7c46b58175d585d9c8040a is 50, key is test_row_0/A:col10/1730972229907/Put/seqid=0 2024-11-07T09:37:09,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742123_1299 (size=12151) 2024-11-07T09:37:09,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/eda6886d0f7c46b58175d585d9c8040a 2024-11-07T09:37:09,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-07T09:37:09,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:09,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42826 deadline: 1730972289939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:09,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/3e19d5ffd7cf4054b608ad07dcad7554 is 50, key is test_row_0/B:col10/1730972229907/Put/seqid=0 2024-11-07T09:37:09,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:09,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1730972289940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:09,945 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:09,945 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:09,945 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:09,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:09,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:09,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:09,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:09,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42842 deadline: 1730972289943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:09,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:09,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:09,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:09,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730972289943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:09,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:09,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:09,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:09,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:09,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742124_1300 (size=12151) 2024-11-07T09:37:09,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-07T09:37:09,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:09,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. as already flushing 2024-11-07T09:37:09,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:09,993 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:09,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:09,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:10,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:10,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1730972290053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:10,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:10,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42826 deadline: 1730972290053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:10,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:10,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:10,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:10,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:10,056 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:10,056 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:10,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730972290053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:10,056 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:10,057 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:10,057 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:10,057 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:10,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:10,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42842 deadline: 1730972290053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:10,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:10,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:10,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:10,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-07T09:37:10,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:10,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. as already flushing 2024-11-07T09:37:10,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:10,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:10,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:10,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:10,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-07T09:37:10,183 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/9c93a880b0464030823dd86c8215b165 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/9c93a880b0464030823dd86c8215b165 2024-11-07T09:37:10,189 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5165da7af32dbdfd119fd1fa7507f91f/B of 5165da7af32dbdfd119fd1fa7507f91f into 9c93a880b0464030823dd86c8215b165(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:37:10,189 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:10,189 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., storeName=5165da7af32dbdfd119fd1fa7507f91f/B, priority=13, startTime=1730972229698; duration=0sec 2024-11-07T09:37:10,189 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:10,189 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5165da7af32dbdfd119fd1fa7507f91f:B 2024-11-07T09:37:10,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:10,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1730972290263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:10,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:10,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:10,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:10,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42826 deadline: 1730972290263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:10,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:10,266 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:10,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:10,267 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:10,267 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:10,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730972290263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:10,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:10,267 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:10,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42842 deadline: 1730972290264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:10,267 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:10,267 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:10,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:10,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:10,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:10,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-07T09:37:10,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:10,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. as already flushing 2024-11-07T09:37:10,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:10,301 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:10,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:10,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:10,359 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/3e19d5ffd7cf4054b608ad07dcad7554 2024-11-07T09:37:10,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/152ec1b809ec4cc3b79f6f1a47926229 is 50, key is test_row_0/C:col10/1730972229907/Put/seqid=0 2024-11-07T09:37:10,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742125_1301 (size=12151) 2024-11-07T09:37:10,377 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/152ec1b809ec4cc3b79f6f1a47926229 2024-11-07T09:37:10,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/eda6886d0f7c46b58175d585d9c8040a as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/eda6886d0f7c46b58175d585d9c8040a 2024-11-07T09:37:10,387 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/eda6886d0f7c46b58175d585d9c8040a, entries=150, sequenceid=214, filesize=11.9 K 2024-11-07T09:37:10,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/3e19d5ffd7cf4054b608ad07dcad7554 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/3e19d5ffd7cf4054b608ad07dcad7554 2024-11-07T09:37:10,392 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/3e19d5ffd7cf4054b608ad07dcad7554, entries=150, sequenceid=214, filesize=11.9 K 2024-11-07T09:37:10,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/152ec1b809ec4cc3b79f6f1a47926229 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/152ec1b809ec4cc3b79f6f1a47926229 2024-11-07T09:37:10,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/152ec1b809ec4cc3b79f6f1a47926229, entries=150, sequenceid=214, filesize=11.9 K 2024-11-07T09:37:10,398 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 5165da7af32dbdfd119fd1fa7507f91f in 491ms, sequenceid=214, compaction requested=false 2024-11-07T09:37:10,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:10,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-07T09:37:10,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:10,455 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 5165da7af32dbdfd119fd1fa7507f91f 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-07T09:37:10,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/d10316279e4b4152944e430b5c2ecc5b is 50, key is test_row_0/A:col10/1730972229916/Put/seqid=0 2024-11-07T09:37:10,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-07T09:37:10,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742126_1302 (size=12151) 2024-11-07T09:37:10,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 5165da7af32dbdfd119fd1fa7507f91f 2024-11-07T09:37:10,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. as already flushing 2024-11-07T09:37:10,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:10,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42826 deadline: 1730972290584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:10,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:10,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42842 deadline: 1730972290584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:10,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:10,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:10,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:10,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:10,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:10,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:10,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:10,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1730972290587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:10,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:10,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:10,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:10,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:10,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972290590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:10,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:10,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:10,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:10,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730972290590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:10,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:10,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:10,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:10,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:10,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:10,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42826 deadline: 1730972290693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:10,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:10,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:10,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:10,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42842 deadline: 1730972290693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:10,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:10,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:10,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:10,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1730972290693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:10,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:10,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:10,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:10,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:10,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:10,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:10,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972290703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:10,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:10,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:10,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:10,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730972290703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:10,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:10,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:10,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:10,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:10,869 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/d10316279e4b4152944e430b5c2ecc5b 2024-11-07T09:37:10,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/75c1345ceb2643f6a979f6c83cc33f8f is 50, key is test_row_0/B:col10/1730972229916/Put/seqid=0 2024-11-07T09:37:10,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742127_1303 (size=12151) 2024-11-07T09:37:10,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:10,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42826 deadline: 1730972290903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:10,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:10,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:10,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:10,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:10,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42842 deadline: 1730972290903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:10,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:10,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1730972290903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:10,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:10,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:10,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:10,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:10,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:10,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:10,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:10,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972290913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:10,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:10,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730972290913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:10,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:10,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:10,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:10,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:10,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:10,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:10,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-07T09:37:11,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:11,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42826 deadline: 1730972291213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:11,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:11,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:11,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:11,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42842 deadline: 1730972291213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:11,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:11,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:11,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:11,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1730972291214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:11,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:11,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:11,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:11,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:11,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:11,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:11,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972291223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:11,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:11,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:11,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:11,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:11,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730972291223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:11,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:11,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:11,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:11,282 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/75c1345ceb2643f6a979f6c83cc33f8f 2024-11-07T09:37:11,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/841b9026c0e145acaa92fecb493e0093 is 50, key is test_row_0/C:col10/1730972229916/Put/seqid=0 2024-11-07T09:37:11,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742128_1304 (size=12151) 2024-11-07T09:37:11,699 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/841b9026c0e145acaa92fecb493e0093 2024-11-07T09:37:11,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/d10316279e4b4152944e430b5c2ecc5b as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/d10316279e4b4152944e430b5c2ecc5b 2024-11-07T09:37:11,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:11,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42826 deadline: 1730972291723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:11,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:11,724 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:11,724 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:11,724 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:11,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1730972291723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:11,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:11,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:11,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:11,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:11,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42842 deadline: 1730972291723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:11,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:11,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:11,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:11,726 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/d10316279e4b4152944e430b5c2ecc5b, entries=150, sequenceid=238, filesize=11.9 K 2024-11-07T09:37:11,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/75c1345ceb2643f6a979f6c83cc33f8f as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/75c1345ceb2643f6a979f6c83cc33f8f 2024-11-07T09:37:11,733 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/75c1345ceb2643f6a979f6c83cc33f8f, entries=150, sequenceid=238, filesize=11.9 K 2024-11-07T09:37:11,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/841b9026c0e145acaa92fecb493e0093 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/841b9026c0e145acaa92fecb493e0093 2024-11-07T09:37:11,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:11,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972291734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:11,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:11,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:11,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:11,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730972291734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:11,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:11,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:11,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:11,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:11,739 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/841b9026c0e145acaa92fecb493e0093, entries=150, sequenceid=238, filesize=11.9 K 2024-11-07T09:37:11,740 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 5165da7af32dbdfd119fd1fa7507f91f in 1285ms, sequenceid=238, compaction requested=true 2024-11-07T09:37:11,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:11,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:11,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-07T09:37:11,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-07T09:37:11,746 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-07T09:37:11,746 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9040 sec 2024-11-07T09:37:11,749 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.9130 sec 2024-11-07T09:37:11,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-07T09:37:11,983 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:37:11,984 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T09:37:11,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-07T09:37:11,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-07T09:37:11,986 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T09:37:11,989 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T09:37:11,989 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T09:37:12,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-07T09:37:12,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-07T09:37:12,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:12,144 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 5165da7af32dbdfd119fd1fa7507f91f 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-07T09:37:12,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/c4a909ffe3284dff91e90db6ae06ccad is 50, key is test_row_0/A:col10/1730972230581/Put/seqid=0 2024-11-07T09:37:12,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742129_1305 (size=12151) 2024-11-07T09:37:12,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-07T09:37:12,556 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/c4a909ffe3284dff91e90db6ae06ccad 2024-11-07T09:37:12,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/eb628937099e4163900f73e085f73d2c is 50, key is test_row_0/B:col10/1730972230581/Put/seqid=0 2024-11-07T09:37:12,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742130_1306 (size=12151) 2024-11-07T09:37:12,570 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/eb628937099e4163900f73e085f73d2c 2024-11-07T09:37:12,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/0da2ebedcd904e11a933d9413d9b13c1 is 50, key is test_row_0/C:col10/1730972230581/Put/seqid=0 2024-11-07T09:37:12,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742131_1307 (size=12151) 2024-11-07T09:37:12,611 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/0da2ebedcd904e11a933d9413d9b13c1 2024-11-07T09:37:12,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-07T09:37:12,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/c4a909ffe3284dff91e90db6ae06ccad as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/c4a909ffe3284dff91e90db6ae06ccad 2024-11-07T09:37:12,637 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/c4a909ffe3284dff91e90db6ae06ccad, entries=150, sequenceid=252, filesize=11.9 K 2024-11-07T09:37:12,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/eb628937099e4163900f73e085f73d2c as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/eb628937099e4163900f73e085f73d2c 2024-11-07T09:37:12,650 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/eb628937099e4163900f73e085f73d2c, entries=150, sequenceid=252, filesize=11.9 K 2024-11-07T09:37:12,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/0da2ebedcd904e11a933d9413d9b13c1 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/0da2ebedcd904e11a933d9413d9b13c1 2024-11-07T09:37:12,663 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/0da2ebedcd904e11a933d9413d9b13c1, entries=150, sequenceid=252, filesize=11.9 K 2024-11-07T09:37:12,664 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=0 B/0 for 5165da7af32dbdfd119fd1fa7507f91f in 520ms, sequenceid=252, compaction requested=true 2024-11-07T09:37:12,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:12,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:12,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-07T09:37:12,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-07T09:37:12,668 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-07T09:37:12,668 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 677 msec 2024-11-07T09:37:12,672 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 687 msec 2024-11-07T09:37:12,757 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:37:12,757 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5165da7af32dbdfd119fd1fa7507f91f 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-07T09:37:12,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 5165da7af32dbdfd119fd1fa7507f91f 2024-11-07T09:37:12,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/d0a4c7369e1c430db8ea11dd53193400 is 50, key is test_row_0/A:col10/1730972232746/Put/seqid=0 2024-11-07T09:37:12,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742132_1308 (size=12301) 2024-11-07T09:37:12,816 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/d0a4c7369e1c430db8ea11dd53193400 2024-11-07T09:37:12,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/702396d74fa542328cc882420c761d92 is 50, key is test_row_0/B:col10/1730972232746/Put/seqid=0 2024-11-07T09:37:12,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742133_1309 (size=12301) 2024-11-07T09:37:12,839 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/702396d74fa542328cc882420c761d92 2024-11-07T09:37:12,849 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/2464a659813545c3bf045e05cf07c653 is 50, key is test_row_0/C:col10/1730972232746/Put/seqid=0 2024-11-07T09:37:12,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742134_1310 (size=12301) 2024-11-07T09:37:12,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:12,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730972292860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:12,867 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:12,867 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:12,867 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:12,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:12,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972292862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:12,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:12,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:12,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:12,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:12,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42826 deadline: 1730972292865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:12,874 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:12,874 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:12,874 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:12,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:12,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42842 deadline: 1730972292866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:12,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:12,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:12,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:12,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:12,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1730972292873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:12,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:12,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:12,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:12,876 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/2464a659813545c3bf045e05cf07c653 2024-11-07T09:37:12,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/d0a4c7369e1c430db8ea11dd53193400 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/d0a4c7369e1c430db8ea11dd53193400 2024-11-07T09:37:12,887 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/d0a4c7369e1c430db8ea11dd53193400, entries=150, sequenceid=265, filesize=12.0 K 2024-11-07T09:37:12,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/702396d74fa542328cc882420c761d92 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/702396d74fa542328cc882420c761d92 2024-11-07T09:37:12,894 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/702396d74fa542328cc882420c761d92, entries=150, sequenceid=265, filesize=12.0 K 2024-11-07T09:37:12,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/2464a659813545c3bf045e05cf07c653 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/2464a659813545c3bf045e05cf07c653 2024-11-07T09:37:12,901 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/2464a659813545c3bf045e05cf07c653, entries=150, sequenceid=265, filesize=12.0 K 2024-11-07T09:37:12,903 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 5165da7af32dbdfd119fd1fa7507f91f in 146ms, sequenceid=265, compaction requested=true 2024-11-07T09:37:12,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:12,903 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-07T09:37:12,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5165da7af32dbdfd119fd1fa7507f91f:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T09:37:12,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:12,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5165da7af32dbdfd119fd1fa7507f91f:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T09:37:12,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:12,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5165da7af32dbdfd119fd1fa7507f91f:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:37:12,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:37:12,903 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-07T09:37:12,910 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61349 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-07T09:37:12,910 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 5165da7af32dbdfd119fd1fa7507f91f/B is initiating minor compaction (all files) 2024-11-07T09:37:12,910 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5165da7af32dbdfd119fd1fa7507f91f/B in TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:12,910 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/9c93a880b0464030823dd86c8215b165, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/3e19d5ffd7cf4054b608ad07dcad7554, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/75c1345ceb2643f6a979f6c83cc33f8f, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/eb628937099e4163900f73e085f73d2c, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/702396d74fa542328cc882420c761d92] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp, totalSize=59.9 K 2024-11-07T09:37:12,910 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61349 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-07T09:37:12,910 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 5165da7af32dbdfd119fd1fa7507f91f/A is initiating minor compaction (all files) 2024-11-07T09:37:12,911 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5165da7af32dbdfd119fd1fa7507f91f/A in TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:12,911 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/fdf4034adfa2457e8df7e60dae72f0d2, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/eda6886d0f7c46b58175d585d9c8040a, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/d10316279e4b4152944e430b5c2ecc5b, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/c4a909ffe3284dff91e90db6ae06ccad, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/d0a4c7369e1c430db8ea11dd53193400] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp, totalSize=59.9 K 2024-11-07T09:37:12,911 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c93a880b0464030823dd86c8215b165, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1730972229554 2024-11-07T09:37:12,911 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting fdf4034adfa2457e8df7e60dae72f0d2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1730972229554 2024-11-07T09:37:12,911 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e19d5ffd7cf4054b608ad07dcad7554, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1730972229901 2024-11-07T09:37:12,912 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting eda6886d0f7c46b58175d585d9c8040a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1730972229901 2024-11-07T09:37:12,912 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 75c1345ceb2643f6a979f6c83cc33f8f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1730972229916 2024-11-07T09:37:12,912 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting d10316279e4b4152944e430b5c2ecc5b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1730972229916 2024-11-07T09:37:12,913 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting eb628937099e4163900f73e085f73d2c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1730972230576 2024-11-07T09:37:12,913 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4a909ffe3284dff91e90db6ae06ccad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1730972230576 2024-11-07T09:37:12,914 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0a4c7369e1c430db8ea11dd53193400, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1730972232746 2024-11-07T09:37:12,914 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 702396d74fa542328cc882420c761d92, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1730972232746 2024-11-07T09:37:12,940 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5165da7af32dbdfd119fd1fa7507f91f#B#compaction#264 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:37:12,941 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/40d337b9c7584b11ba168e64a1c589f4 is 50, key is test_row_0/B:col10/1730972232746/Put/seqid=0 2024-11-07T09:37:12,947 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5165da7af32dbdfd119fd1fa7507f91f#A#compaction#265 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:37:12,948 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/39a8040f3bfe45dd84059e201d5380e6 is 50, key is test_row_0/A:col10/1730972232746/Put/seqid=0 2024-11-07T09:37:12,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 5165da7af32dbdfd119fd1fa7507f91f 2024-11-07T09:37:12,980 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:37:12,980 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5165da7af32dbdfd119fd1fa7507f91f 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T09:37:12,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742135_1311 (size=12915) 2024-11-07T09:37:12,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/63471011cc884ce4b60054549d803cdb is 50, key is test_row_0/A:col10/1730972232864/Put/seqid=0 2024-11-07T09:37:13,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:13,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42826 deadline: 1730972292992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:13,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:13,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42842 deadline: 1730972292993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:13,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:13,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:13,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:13,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742136_1312 (size=12915) 2024-11-07T09:37:13,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:13,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:13,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:13,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:13,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1730972293002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:13,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:13,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:13,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:13,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:13,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730972293003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:13,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:13,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:13,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:13,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:13,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972293003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:13,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:13,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:13,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:13,016 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/39a8040f3bfe45dd84059e201d5380e6 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/39a8040f3bfe45dd84059e201d5380e6 2024-11-07T09:37:13,024 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 5165da7af32dbdfd119fd1fa7507f91f/A of 5165da7af32dbdfd119fd1fa7507f91f into 39a8040f3bfe45dd84059e201d5380e6(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:37:13,025 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:13,025 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., storeName=5165da7af32dbdfd119fd1fa7507f91f/A, priority=11, startTime=1730972232903; duration=0sec 2024-11-07T09:37:13,025 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:37:13,025 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5165da7af32dbdfd119fd1fa7507f91f:A 2024-11-07T09:37:13,025 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-07T09:37:13,026 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61349 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-07T09:37:13,026 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 5165da7af32dbdfd119fd1fa7507f91f/C is initiating minor compaction (all files) 2024-11-07T09:37:13,027 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5165da7af32dbdfd119fd1fa7507f91f/C in TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:13,027 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/aab565688d664ead8cfdb44eb9d6a626, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/152ec1b809ec4cc3b79f6f1a47926229, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/841b9026c0e145acaa92fecb493e0093, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/0da2ebedcd904e11a933d9413d9b13c1, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/2464a659813545c3bf045e05cf07c653] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp, totalSize=59.9 K 2024-11-07T09:37:13,027 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting aab565688d664ead8cfdb44eb9d6a626, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1730972229554 2024-11-07T09:37:13,028 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 152ec1b809ec4cc3b79f6f1a47926229, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1730972229901 2024-11-07T09:37:13,028 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 841b9026c0e145acaa92fecb493e0093, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1730972229916 2024-11-07T09:37:13,029 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0da2ebedcd904e11a933d9413d9b13c1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1730972230576 2024-11-07T09:37:13,030 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2464a659813545c3bf045e05cf07c653, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1730972232746 2024-11-07T09:37:13,056 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5165da7af32dbdfd119fd1fa7507f91f#C#compaction#267 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:37:13,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742137_1313 (size=14741) 2024-11-07T09:37:13,056 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/f1c28f9726d641a7bd453d2d3ccdfe7a is 50, key is test_row_0/C:col10/1730972232746/Put/seqid=0 2024-11-07T09:37:13,058 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/63471011cc884ce4b60054549d803cdb 2024-11-07T09:37:13,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742138_1314 (size=12915) 2024-11-07T09:37:13,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/390844df72af4820bb5d935815e0f4c2 is 50, key is test_row_0/B:col10/1730972232864/Put/seqid=0 2024-11-07T09:37:13,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:13,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42826 deadline: 1730972293113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:13,118 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:13,118 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:13,118 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:13,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:13,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42842 deadline: 1730972293113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:13,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:13,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:13,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:13,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-07T09:37:13,123 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:37:13,124 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T09:37:13,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-07T09:37:13,126 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T09:37:13,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-07T09:37:13,127 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T09:37:13,127 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T09:37:13,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:13,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972293123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:13,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:13,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:13,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:13,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:13,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1730972293123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:13,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:13,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:13,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:13,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:13,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730972293124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:13,132 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:13,132 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:13,132 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:13,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742139_1315 (size=12301) 2024-11-07T09:37:13,146 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/390844df72af4820bb5d935815e0f4c2 2024-11-07T09:37:13,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/3b7019ff7dde4051a6ed0336eafdce10 is 50, key is test_row_0/C:col10/1730972232864/Put/seqid=0 2024-11-07T09:37:13,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742140_1316 (size=12301) 2024-11-07T09:37:13,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-07T09:37:13,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-07T09:37:13,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:13,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. as already flushing 2024-11-07T09:37:13,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:13,281 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:13,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:13,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:13,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:13,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42826 deadline: 1730972293323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:13,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:13,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:13,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:13,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:13,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42842 deadline: 1730972293323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:13,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:13,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:13,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:13,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:13,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972293343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:13,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:13,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:13,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1730972293343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:13,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:13,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:13,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:13,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:13,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:13,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:13,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730972293343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:13,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:13,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:13,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:13,398 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/40d337b9c7584b11ba168e64a1c589f4 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/40d337b9c7584b11ba168e64a1c589f4 2024-11-07T09:37:13,405 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 5165da7af32dbdfd119fd1fa7507f91f/B of 5165da7af32dbdfd119fd1fa7507f91f into 40d337b9c7584b11ba168e64a1c589f4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:37:13,405 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:13,405 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., storeName=5165da7af32dbdfd119fd1fa7507f91f/B, priority=11, startTime=1730972232903; duration=0sec 2024-11-07T09:37:13,405 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:13,405 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5165da7af32dbdfd119fd1fa7507f91f:B 2024-11-07T09:37:13,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-07T09:37:13,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:13,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. as already flushing 2024-11-07T09:37:13,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:13,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:13,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:13,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-07T09:37:13,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:13,504 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/f1c28f9726d641a7bd453d2d3ccdfe7a as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/f1c28f9726d641a7bd453d2d3ccdfe7a 2024-11-07T09:37:13,509 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 5165da7af32dbdfd119fd1fa7507f91f/C of 5165da7af32dbdfd119fd1fa7507f91f into f1c28f9726d641a7bd453d2d3ccdfe7a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:37:13,510 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:13,510 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., storeName=5165da7af32dbdfd119fd1fa7507f91f/C, priority=11, startTime=1730972232903; duration=0sec 2024-11-07T09:37:13,510 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:13,510 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5165da7af32dbdfd119fd1fa7507f91f:C 2024-11-07T09:37:13,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-07T09:37:13,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:13,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. as already flushing 2024-11-07T09:37:13,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:13,599 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:13,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:13,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:13,628 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/3b7019ff7dde4051a6ed0336eafdce10 2024-11-07T09:37:13,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/63471011cc884ce4b60054549d803cdb as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/63471011cc884ce4b60054549d803cdb 2024-11-07T09:37:13,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/63471011cc884ce4b60054549d803cdb, entries=200, sequenceid=290, filesize=14.4 K 2024-11-07T09:37:13,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/390844df72af4820bb5d935815e0f4c2 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/390844df72af4820bb5d935815e0f4c2 2024-11-07T09:37:13,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:13,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42826 deadline: 1730972293643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:13,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:13,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:13,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:13,644 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/390844df72af4820bb5d935815e0f4c2, entries=150, sequenceid=290, filesize=12.0 K 2024-11-07T09:37:13,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/3b7019ff7dde4051a6ed0336eafdce10 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/3b7019ff7dde4051a6ed0336eafdce10 2024-11-07T09:37:13,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:13,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42842 deadline: 1730972293644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:13,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:13,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:13,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:13,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:13,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730972293653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:13,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:13,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:13,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:13,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:13,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972293656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:13,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:13,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:13,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:13,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:13,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1730972293656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:13,658 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/3b7019ff7dde4051a6ed0336eafdce10, entries=150, sequenceid=290, filesize=12.0 K 2024-11-07T09:37:13,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:13,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:13,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:13,659 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 5165da7af32dbdfd119fd1fa7507f91f in 679ms, sequenceid=290, compaction requested=false 2024-11-07T09:37:13,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:13,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-07T09:37:13,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-07T09:37:13,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:13,755 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing 5165da7af32dbdfd119fd1fa7507f91f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T09:37:13,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/66951a2517e843998a292a4f6ef4f7df is 50, key is test_row_0/A:col10/1730972232999/Put/seqid=0 2024-11-07T09:37:13,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742141_1317 (size=12301) 2024-11-07T09:37:13,811 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/66951a2517e843998a292a4f6ef4f7df 2024-11-07T09:37:13,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/1681f95fbb5141769501952149b07a48 is 50, key is test_row_0/B:col10/1730972232999/Put/seqid=0 2024-11-07T09:37:13,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742142_1318 (size=12301) 2024-11-07T09:37:13,862 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/1681f95fbb5141769501952149b07a48 2024-11-07T09:37:13,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/486a4ce51546400c96bbf2a52eb59e8e is 50, key is test_row_0/C:col10/1730972232999/Put/seqid=0 2024-11-07T09:37:13,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742143_1319 (size=12301) 2024-11-07T09:37:14,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 5165da7af32dbdfd119fd1fa7507f91f 2024-11-07T09:37:14,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. as already flushing 2024-11-07T09:37:14,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:14,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1730972294181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:14,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:14,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:14,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:14,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:14,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42826 deadline: 1730972294182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:14,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:14,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:14,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:14,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:14,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42842 deadline: 1730972294183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:14,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:14,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:14,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730972294185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:14,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:14,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:14,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:14,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:14,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:14,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:14,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972294187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:14,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:14,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:14,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:14,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-07T09:37:14,283 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/486a4ce51546400c96bbf2a52eb59e8e 2024-11-07T09:37:14,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/66951a2517e843998a292a4f6ef4f7df as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/66951a2517e843998a292a4f6ef4f7df 2024-11-07T09:37:14,291 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/66951a2517e843998a292a4f6ef4f7df, entries=150, sequenceid=304, filesize=12.0 K 2024-11-07T09:37:14,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/1681f95fbb5141769501952149b07a48 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/1681f95fbb5141769501952149b07a48 2024-11-07T09:37:14,297 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/1681f95fbb5141769501952149b07a48, entries=150, sequenceid=304, filesize=12.0 K 2024-11-07T09:37:14,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:14,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1730972294292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:14,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:14,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:14,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:14,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:14,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:14,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42842 deadline: 1730972294293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:14,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42826 deadline: 1730972294293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:14,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:14,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:14,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:14,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:14,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730972294293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:14,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:14,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:14,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:14,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:14,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:14,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:14,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/486a4ce51546400c96bbf2a52eb59e8e as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/486a4ce51546400c96bbf2a52eb59e8e 2024-11-07T09:37:14,302 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/486a4ce51546400c96bbf2a52eb59e8e, entries=150, sequenceid=304, filesize=12.0 K 2024-11-07T09:37:14,302 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 5165da7af32dbdfd119fd1fa7507f91f in 547ms, sequenceid=304, compaction requested=true 2024-11-07T09:37:14,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:14,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:14,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-07T09:37:14,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-07T09:37:14,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 5165da7af32dbdfd119fd1fa7507f91f 2024-11-07T09:37:14,305 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:37:14,305 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5165da7af32dbdfd119fd1fa7507f91f 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-07T09:37:14,306 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-07T09:37:14,306 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1770 sec 2024-11-07T09:37:14,308 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.1830 sec 2024-11-07T09:37:14,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/3cba0eee27b143a091e8bec2967076da is 50, key is test_row_0/A:col10/1730972234186/Put/seqid=0 2024-11-07T09:37:14,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742144_1320 (size=14741) 2024-11-07T09:37:14,344 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/3cba0eee27b143a091e8bec2967076da 2024-11-07T09:37:14,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:14,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972294342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:14,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:14,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:14,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:14,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/a0ab9018fa414d0aace2ce4bb9d8da38 is 50, key is test_row_0/B:col10/1730972234186/Put/seqid=0 2024-11-07T09:37:14,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742145_1321 (size=12301) 2024-11-07T09:37:14,429 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/a0ab9018fa414d0aace2ce4bb9d8da38 2024-11-07T09:37:14,453 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/8707b54ccd234dc7b81e0b0e47da7f50 is 50, key is test_row_0/C:col10/1730972234186/Put/seqid=0 2024-11-07T09:37:14,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:14,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972294458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:14,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:14,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:14,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:14,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742146_1322 (size=12301) 2024-11-07T09:37:14,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:14,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1730972294504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:14,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:14,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42826 deadline: 1730972294504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:14,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:14,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:14,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730972294505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:14,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:14,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:14,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:14,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:14,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:14,508 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:14,508 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:14,508 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:14,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:14,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42842 deadline: 1730972294505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:14,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:14,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:14,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:14,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:14,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972294663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:14,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:14,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:14,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:14,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:14,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42826 deadline: 1730972294813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:14,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:14,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:14,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:14,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:14,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1730972294813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:14,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:14,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:14,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:14,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730972294814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:14,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:14,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:14,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:14,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42842 deadline: 1730972294814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:14,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:14,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:14,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:14,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:14,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:14,881 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/8707b54ccd234dc7b81e0b0e47da7f50 2024-11-07T09:37:14,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/3cba0eee27b143a091e8bec2967076da as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/3cba0eee27b143a091e8bec2967076da 2024-11-07T09:37:14,899 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/3cba0eee27b143a091e8bec2967076da, entries=200, sequenceid=330, filesize=14.4 K 2024-11-07T09:37:14,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/a0ab9018fa414d0aace2ce4bb9d8da38 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/a0ab9018fa414d0aace2ce4bb9d8da38 2024-11-07T09:37:14,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/a0ab9018fa414d0aace2ce4bb9d8da38, entries=150, sequenceid=330, filesize=12.0 K 2024-11-07T09:37:14,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/8707b54ccd234dc7b81e0b0e47da7f50 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/8707b54ccd234dc7b81e0b0e47da7f50 2024-11-07T09:37:14,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/8707b54ccd234dc7b81e0b0e47da7f50, entries=150, sequenceid=330, filesize=12.0 K 2024-11-07T09:37:14,930 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 5165da7af32dbdfd119fd1fa7507f91f in 624ms, sequenceid=330, compaction requested=true 2024-11-07T09:37:14,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:14,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5165da7af32dbdfd119fd1fa7507f91f:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T09:37:14,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:14,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5165da7af32dbdfd119fd1fa7507f91f:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T09:37:14,930 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T09:37:14,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:14,930 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T09:37:14,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5165da7af32dbdfd119fd1fa7507f91f:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:37:14,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:37:14,931 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T09:37:14,931 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 5165da7af32dbdfd119fd1fa7507f91f/B is initiating minor compaction (all files) 2024-11-07T09:37:14,931 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54698 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T09:37:14,931 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 5165da7af32dbdfd119fd1fa7507f91f/A is initiating minor compaction (all files) 2024-11-07T09:37:14,931 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5165da7af32dbdfd119fd1fa7507f91f/B in TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:14,931 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5165da7af32dbdfd119fd1fa7507f91f/A in TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:14,932 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/39a8040f3bfe45dd84059e201d5380e6, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/63471011cc884ce4b60054549d803cdb, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/66951a2517e843998a292a4f6ef4f7df, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/3cba0eee27b143a091e8bec2967076da] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp, totalSize=53.4 K 2024-11-07T09:37:14,932 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/40d337b9c7584b11ba168e64a1c589f4, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/390844df72af4820bb5d935815e0f4c2, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/1681f95fbb5141769501952149b07a48, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/a0ab9018fa414d0aace2ce4bb9d8da38] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp, totalSize=48.7 K 2024-11-07T09:37:14,932 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39a8040f3bfe45dd84059e201d5380e6, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1730972232746 2024-11-07T09:37:14,932 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 40d337b9c7584b11ba168e64a1c589f4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1730972232746 2024-11-07T09:37:14,933 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 390844df72af4820bb5d935815e0f4c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1730972232864 2024-11-07T09:37:14,933 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63471011cc884ce4b60054549d803cdb, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1730972232860 2024-11-07T09:37:14,933 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 1681f95fbb5141769501952149b07a48, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1730972232995 2024-11-07T09:37:14,933 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66951a2517e843998a292a4f6ef4f7df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1730972232995 2024-11-07T09:37:14,934 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3cba0eee27b143a091e8bec2967076da, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1730972234175 2024-11-07T09:37:14,934 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting a0ab9018fa414d0aace2ce4bb9d8da38, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1730972234181 2024-11-07T09:37:14,955 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5165da7af32dbdfd119fd1fa7507f91f#A#compaction#276 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:37:14,956 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/03c58a40343249e7af5596a53b222a21 is 50, key is test_row_0/A:col10/1730972234186/Put/seqid=0 2024-11-07T09:37:14,974 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5165da7af32dbdfd119fd1fa7507f91f#B#compaction#277 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:37:14,975 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/1a9e4432cc524e5daaadf24274ce3e11 is 50, key is test_row_0/B:col10/1730972234186/Put/seqid=0 2024-11-07T09:37:14,976 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:37:14,976 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5165da7af32dbdfd119fd1fa7507f91f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T09:37:14,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 5165da7af32dbdfd119fd1fa7507f91f 2024-11-07T09:37:14,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/a3e80b4e1f5e4ab3abc24e84c3f54225 is 50, key is test_row_0/A:col10/1730972234974/Put/seqid=0 2024-11-07T09:37:15,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742147_1323 (size=13051) 2024-11-07T09:37:15,013 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/03c58a40343249e7af5596a53b222a21 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/03c58a40343249e7af5596a53b222a21 2024-11-07T09:37:15,020 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5165da7af32dbdfd119fd1fa7507f91f/A of 5165da7af32dbdfd119fd1fa7507f91f into 03c58a40343249e7af5596a53b222a21(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:37:15,020 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:15,020 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., storeName=5165da7af32dbdfd119fd1fa7507f91f/A, priority=12, startTime=1730972234930; duration=0sec 2024-11-07T09:37:15,020 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:37:15,020 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5165da7af32dbdfd119fd1fa7507f91f:A 2024-11-07T09:37:15,020 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T09:37:15,022 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T09:37:15,022 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 5165da7af32dbdfd119fd1fa7507f91f/C is initiating minor compaction (all files) 2024-11-07T09:37:15,022 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5165da7af32dbdfd119fd1fa7507f91f/C in TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:15,022 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/f1c28f9726d641a7bd453d2d3ccdfe7a, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/3b7019ff7dde4051a6ed0336eafdce10, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/486a4ce51546400c96bbf2a52eb59e8e, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/8707b54ccd234dc7b81e0b0e47da7f50] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp, totalSize=48.7 K 2024-11-07T09:37:15,022 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1c28f9726d641a7bd453d2d3ccdfe7a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1730972232746 2024-11-07T09:37:15,023 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b7019ff7dde4051a6ed0336eafdce10, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1730972232864 2024-11-07T09:37:15,024 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 486a4ce51546400c96bbf2a52eb59e8e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1730972232995 2024-11-07T09:37:15,025 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8707b54ccd234dc7b81e0b0e47da7f50, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1730972234181 2024-11-07T09:37:15,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742148_1324 (size=13051) 2024-11-07T09:37:15,040 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/1a9e4432cc524e5daaadf24274ce3e11 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/1a9e4432cc524e5daaadf24274ce3e11 2024-11-07T09:37:15,048 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5165da7af32dbdfd119fd1fa7507f91f/B of 5165da7af32dbdfd119fd1fa7507f91f into 1a9e4432cc524e5daaadf24274ce3e11(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:37:15,049 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:15,049 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., storeName=5165da7af32dbdfd119fd1fa7507f91f/B, priority=12, startTime=1730972234930; duration=0sec 2024-11-07T09:37:15,049 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:15,049 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5165da7af32dbdfd119fd1fa7507f91f:B 2024-11-07T09:37:15,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742149_1325 (size=12297) 2024-11-07T09:37:15,059 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/a3e80b4e1f5e4ab3abc24e84c3f54225 2024-11-07T09:37:15,063 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5165da7af32dbdfd119fd1fa7507f91f#C#compaction#279 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:37:15,064 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/36321d6abbb9438aa47660e166c92be0 is 50, key is test_row_0/C:col10/1730972234186/Put/seqid=0 2024-11-07T09:37:15,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/f9b223ebc2fd4c6c8f54ddfcc40cc01e is 50, key is test_row_0/B:col10/1730972234974/Put/seqid=0 2024-11-07T09:37:15,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742150_1326 (size=13051) 2024-11-07T09:37:15,123 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/36321d6abbb9438aa47660e166c92be0 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/36321d6abbb9438aa47660e166c92be0 2024-11-07T09:37:15,128 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5165da7af32dbdfd119fd1fa7507f91f/C of 5165da7af32dbdfd119fd1fa7507f91f into 36321d6abbb9438aa47660e166c92be0(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:37:15,128 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:15,128 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., storeName=5165da7af32dbdfd119fd1fa7507f91f/C, priority=12, startTime=1730972234930; duration=0sec 2024-11-07T09:37:15,128 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:15,128 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5165da7af32dbdfd119fd1fa7507f91f:C 2024-11-07T09:37:15,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:15,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972295132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:15,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:15,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:15,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:15,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742151_1327 (size=9857) 2024-11-07T09:37:15,152 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/f9b223ebc2fd4c6c8f54ddfcc40cc01e 2024-11-07T09:37:15,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/d97fecfee8134c539dfa942761827187 is 50, key is test_row_0/C:col10/1730972234974/Put/seqid=0 2024-11-07T09:37:15,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742152_1328 (size=9857) 2024-11-07T09:37:15,193 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/d97fecfee8134c539dfa942761827187 2024-11-07T09:37:15,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/a3e80b4e1f5e4ab3abc24e84c3f54225 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/a3e80b4e1f5e4ab3abc24e84c3f54225 2024-11-07T09:37:15,206 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/a3e80b4e1f5e4ab3abc24e84c3f54225, entries=150, sequenceid=341, filesize=12.0 K 2024-11-07T09:37:15,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/f9b223ebc2fd4c6c8f54ddfcc40cc01e as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/f9b223ebc2fd4c6c8f54ddfcc40cc01e 2024-11-07T09:37:15,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/f9b223ebc2fd4c6c8f54ddfcc40cc01e, entries=100, sequenceid=341, filesize=9.6 K 2024-11-07T09:37:15,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/d97fecfee8134c539dfa942761827187 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/d97fecfee8134c539dfa942761827187 2024-11-07T09:37:15,228 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/d97fecfee8134c539dfa942761827187, entries=100, sequenceid=341, filesize=9.6 K 2024-11-07T09:37:15,230 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 5165da7af32dbdfd119fd1fa7507f91f in 254ms, sequenceid=341, compaction requested=false 2024-11-07T09:37:15,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:15,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 5165da7af32dbdfd119fd1fa7507f91f 2024-11-07T09:37:15,248 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:37:15,248 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5165da7af32dbdfd119fd1fa7507f91f 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-07T09:37:15,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/90e1e75c2a7e40fd81c5af551b6d67c5 is 50, key is test_row_0/A:col10/1730972235245/Put/seqid=0 2024-11-07T09:37:15,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742153_1329 (size=14741) 2024-11-07T09:37:15,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/90e1e75c2a7e40fd81c5af551b6d67c5 2024-11-07T09:37:15,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-07T09:37:15,274 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:37:15,275 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T09:37:15,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-07T09:37:15,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-07T09:37:15,277 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T09:37:15,278 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T09:37:15,278 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T09:37:15,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/0d705be7a0d942e196f8cd0fe2f2ac6f is 50, key is test_row_0/B:col10/1730972235245/Put/seqid=0 2024-11-07T09:37:15,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742154_1330 (size=12301) 2024-11-07T09:37:15,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:15,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972295307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:15,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:15,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:15,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:15,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:15,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42826 deadline: 1730972295323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:15,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:15,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:15,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:15,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:15,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730972295323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:15,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:15,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:15,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:15,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1730972295323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:15,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:15,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:15,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:15,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:15,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:15,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42842 deadline: 1730972295332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:15,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:15,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:15,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:15,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-07T09:37:15,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:15,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972295423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:15,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:15,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:15,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:15,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-07T09:37:15,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:15,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. as already flushing 2024-11-07T09:37:15,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:15,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:15,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:15,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:15,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-07T09:37:15,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:15,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. as already flushing 2024-11-07T09:37:15,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:15,590 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:15,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:15,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:15,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-07T09:37:15,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:15,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972295632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:15,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:15,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:15,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:15,713 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/0d705be7a0d942e196f8cd0fe2f2ac6f 2024-11-07T09:37:15,720 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/a780b9b7ca0648fc821759ab3ced8fc1 is 50, key is test_row_0/C:col10/1730972235245/Put/seqid=0 2024-11-07T09:37:15,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742155_1331 (size=12301) 2024-11-07T09:37:15,726 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/a780b9b7ca0648fc821759ab3ced8fc1 2024-11-07T09:37:15,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/90e1e75c2a7e40fd81c5af551b6d67c5 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/90e1e75c2a7e40fd81c5af551b6d67c5 2024-11-07T09:37:15,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/90e1e75c2a7e40fd81c5af551b6d67c5, entries=200, sequenceid=370, filesize=14.4 K 2024-11-07T09:37:15,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/0d705be7a0d942e196f8cd0fe2f2ac6f as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/0d705be7a0d942e196f8cd0fe2f2ac6f 2024-11-07T09:37:15,741 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/0d705be7a0d942e196f8cd0fe2f2ac6f, entries=150, sequenceid=370, filesize=12.0 K 2024-11-07T09:37:15,743 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-07T09:37:15,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:15,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. as already flushing 2024-11-07T09:37:15,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:15,743 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:15,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:15,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/a780b9b7ca0648fc821759ab3ced8fc1 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/a780b9b7ca0648fc821759ab3ced8fc1 2024-11-07T09:37:15,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:15,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/a780b9b7ca0648fc821759ab3ced8fc1, entries=150, sequenceid=370, filesize=12.0 K 2024-11-07T09:37:15,750 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 5165da7af32dbdfd119fd1fa7507f91f in 502ms, sequenceid=370, compaction requested=true 2024-11-07T09:37:15,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:15,750 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:37:15,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5165da7af32dbdfd119fd1fa7507f91f:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T09:37:15,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:15,750 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:37:15,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5165da7af32dbdfd119fd1fa7507f91f:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T09:37:15,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:15,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5165da7af32dbdfd119fd1fa7507f91f:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:37:15,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:37:15,751 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40089 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:37:15,751 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 5165da7af32dbdfd119fd1fa7507f91f/A is initiating minor compaction (all files) 2024-11-07T09:37:15,751 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5165da7af32dbdfd119fd1fa7507f91f/A in TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:15,751 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:37:15,751 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/03c58a40343249e7af5596a53b222a21, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/a3e80b4e1f5e4ab3abc24e84c3f54225, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/90e1e75c2a7e40fd81c5af551b6d67c5] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp, totalSize=39.1 K 2024-11-07T09:37:15,751 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 5165da7af32dbdfd119fd1fa7507f91f/B is initiating minor compaction (all files) 2024-11-07T09:37:15,751 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5165da7af32dbdfd119fd1fa7507f91f/B in TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:15,752 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/1a9e4432cc524e5daaadf24274ce3e11, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/f9b223ebc2fd4c6c8f54ddfcc40cc01e, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/0d705be7a0d942e196f8cd0fe2f2ac6f] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp, totalSize=34.4 K 2024-11-07T09:37:15,752 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03c58a40343249e7af5596a53b222a21, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1730972234181 2024-11-07T09:37:15,752 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a9e4432cc524e5daaadf24274ce3e11, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1730972234181 2024-11-07T09:37:15,752 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting f9b223ebc2fd4c6c8f54ddfcc40cc01e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1730972234333 2024-11-07T09:37:15,753 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3e80b4e1f5e4ab3abc24e84c3f54225, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1730972234333 2024-11-07T09:37:15,754 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d705be7a0d942e196f8cd0fe2f2ac6f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1730972235116 2024-11-07T09:37:15,755 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90e1e75c2a7e40fd81c5af551b6d67c5, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1730972235116 2024-11-07T09:37:15,766 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5165da7af32dbdfd119fd1fa7507f91f#B#compaction#285 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:37:15,767 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/c64b40607a12478fa446ce8a13076a03 is 50, key is test_row_0/B:col10/1730972235245/Put/seqid=0 2024-11-07T09:37:15,767 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5165da7af32dbdfd119fd1fa7507f91f#A#compaction#286 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:37:15,768 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/094833d6b09d47cda6cfc4efc49f9ec1 is 50, key is test_row_0/A:col10/1730972235245/Put/seqid=0 2024-11-07T09:37:15,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742156_1332 (size=13153) 2024-11-07T09:37:15,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742157_1333 (size=13153) 2024-11-07T09:37:15,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-07T09:37:15,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:15,897 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing 5165da7af32dbdfd119fd1fa7507f91f 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-07T09:37:15,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-07T09:37:15,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/80d2176ea5f943ca96e7d4255f4b3b95 is 50, key is test_row_0/A:col10/1730972235297/Put/seqid=0 2024-11-07T09:37:15,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742158_1334 (size=12301) 2024-11-07T09:37:15,934 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/80d2176ea5f943ca96e7d4255f4b3b95 2024-11-07T09:37:15,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/652283d14dba47c5ab4dfe832a3d8a03 is 50, key is test_row_0/B:col10/1730972235297/Put/seqid=0 2024-11-07T09:37:15,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 5165da7af32dbdfd119fd1fa7507f91f 2024-11-07T09:37:15,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. as already flushing 2024-11-07T09:37:15,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742159_1335 (size=12301) 2024-11-07T09:37:15,948 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/652283d14dba47c5ab4dfe832a3d8a03 2024-11-07T09:37:15,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/4004364b515b47e181b77c7f16fdd7ef is 50, key is test_row_0/C:col10/1730972235297/Put/seqid=0 2024-11-07T09:37:15,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742160_1336 (size=12301) 2024-11-07T09:37:15,997 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/4004364b515b47e181b77c7f16fdd7ef 2024-11-07T09:37:16,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/80d2176ea5f943ca96e7d4255f4b3b95 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/80d2176ea5f943ca96e7d4255f4b3b95 2024-11-07T09:37:16,007 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/80d2176ea5f943ca96e7d4255f4b3b95, entries=150, sequenceid=380, filesize=12.0 K 2024-11-07T09:37:16,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/652283d14dba47c5ab4dfe832a3d8a03 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/652283d14dba47c5ab4dfe832a3d8a03 2024-11-07T09:37:16,016 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/652283d14dba47c5ab4dfe832a3d8a03, entries=150, sequenceid=380, filesize=12.0 K 2024-11-07T09:37:16,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/4004364b515b47e181b77c7f16fdd7ef as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/4004364b515b47e181b77c7f16fdd7ef 2024-11-07T09:37:16,023 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/4004364b515b47e181b77c7f16fdd7ef, entries=150, sequenceid=380, filesize=12.0 K 2024-11-07T09:37:16,024 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=73.80 KB/75570 for 5165da7af32dbdfd119fd1fa7507f91f in 127ms, sequenceid=380, compaction requested=true 2024-11-07T09:37:16,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:16,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:16,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 5165da7af32dbdfd119fd1fa7507f91f 2024-11-07T09:37:16,025 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:37:16,025 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5165da7af32dbdfd119fd1fa7507f91f 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-07T09:37:16,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-07T09:37:16,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-07T09:37:16,029 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-07T09:37:16,029 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 749 msec 2024-11-07T09:37:16,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/1d6bb8ebbcb1440ca49c1687b38099b4 is 50, key is test_row_0/A:col10/1730972236022/Put/seqid=0 2024-11-07T09:37:16,032 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 755 msec 2024-11-07T09:37:16,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742161_1337 (size=14741) 2024-11-07T09:37:16,058 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/1d6bb8ebbcb1440ca49c1687b38099b4 2024-11-07T09:37:16,076 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/91d8a240b2a944f7b37ed0bb54601cde is 50, key is test_row_0/B:col10/1730972236022/Put/seqid=0 2024-11-07T09:37:16,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742162_1338 (size=12301) 2024-11-07T09:37:16,082 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/91d8a240b2a944f7b37ed0bb54601cde 2024-11-07T09:37:16,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/e85f51f60e7f468b8ff0449c2594e95f is 50, key is test_row_0/C:col10/1730972236022/Put/seqid=0 2024-11-07T09:37:16,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742163_1339 (size=12301) 2024-11-07T09:37:16,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:16,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972296166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:16,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:16,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:16,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:16,200 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/c64b40607a12478fa446ce8a13076a03 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/c64b40607a12478fa446ce8a13076a03 2024-11-07T09:37:16,202 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/094833d6b09d47cda6cfc4efc49f9ec1 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/094833d6b09d47cda6cfc4efc49f9ec1 2024-11-07T09:37:16,209 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5165da7af32dbdfd119fd1fa7507f91f/B of 5165da7af32dbdfd119fd1fa7507f91f into c64b40607a12478fa446ce8a13076a03(size=12.8 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:37:16,209 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:16,209 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., storeName=5165da7af32dbdfd119fd1fa7507f91f/B, priority=13, startTime=1730972235750; duration=0sec 2024-11-07T09:37:16,209 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5165da7af32dbdfd119fd1fa7507f91f/A of 5165da7af32dbdfd119fd1fa7507f91f into 094833d6b09d47cda6cfc4efc49f9ec1(size=12.8 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:37:16,209 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:37:16,209 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5165da7af32dbdfd119fd1fa7507f91f:B 2024-11-07T09:37:16,209 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:16,209 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., storeName=5165da7af32dbdfd119fd1fa7507f91f/A, priority=13, startTime=1730972235750; duration=0sec 2024-11-07T09:37:16,209 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T09:37:16,209 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:16,209 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5165da7af32dbdfd119fd1fa7507f91f:A 2024-11-07T09:37:16,210 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T09:37:16,210 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 5165da7af32dbdfd119fd1fa7507f91f/C is initiating minor compaction (all files) 2024-11-07T09:37:16,211 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5165da7af32dbdfd119fd1fa7507f91f/C in TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:16,211 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/36321d6abbb9438aa47660e166c92be0, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/d97fecfee8134c539dfa942761827187, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/a780b9b7ca0648fc821759ab3ced8fc1, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/4004364b515b47e181b77c7f16fdd7ef] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp, totalSize=46.4 K 2024-11-07T09:37:16,211 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 36321d6abbb9438aa47660e166c92be0, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1730972234181 2024-11-07T09:37:16,212 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting d97fecfee8134c539dfa942761827187, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1730972234333 2024-11-07T09:37:16,212 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting a780b9b7ca0648fc821759ab3ced8fc1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1730972235116 2024-11-07T09:37:16,212 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 4004364b515b47e181b77c7f16fdd7ef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1730972235287 2024-11-07T09:37:16,231 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5165da7af32dbdfd119fd1fa7507f91f#C#compaction#293 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:37:16,232 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/afdb7b58ec924e8caa3f813861afe2ad is 50, key is test_row_0/C:col10/1730972235297/Put/seqid=0 2024-11-07T09:37:16,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742164_1340 (size=13187) 2024-11-07T09:37:16,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:16,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972296273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:16,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:16,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:16,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:16,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:16,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42826 deadline: 1730972296333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:16,335 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 9f1eb259a29b,42895,1730972153970 for 'test_row_2' in 5165da7af32dbdfd119fd1fa7507f91f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T09:37:16,336 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:16,336 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:16,336 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:16,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:16,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730972296343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:16,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:16,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1730972296343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:16,347 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 9f1eb259a29b,42895,1730972153970 for 'test_row_2' in 5165da7af32dbdfd119fd1fa7507f91f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T09:37:16,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:16,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:16,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:16,347 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 9f1eb259a29b,42895,1730972153970 for 'test_row_0' in 5165da7af32dbdfd119fd1fa7507f91f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T09:37:16,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:16,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:16,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:16,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:16,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42842 deadline: 1730972296353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:16,356 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 9f1eb259a29b,42895,1730972153970 for 'test_row_2' in 5165da7af32dbdfd119fd1fa7507f91f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T09:37:16,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:16,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:16,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:16,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-07T09:37:16,413 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:37:16,413 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T09:37:16,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees 2024-11-07T09:37:16,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-07T09:37:16,415 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T09:37:16,416 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T09:37:16,417 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T09:37:16,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:16,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972296483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:16,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:16,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:16,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:16,494 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/e85f51f60e7f468b8ff0449c2594e95f 2024-11-07T09:37:16,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/1d6bb8ebbcb1440ca49c1687b38099b4 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/1d6bb8ebbcb1440ca49c1687b38099b4 2024-11-07T09:37:16,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/1d6bb8ebbcb1440ca49c1687b38099b4, entries=200, sequenceid=395, filesize=14.4 K 2024-11-07T09:37:16,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/91d8a240b2a944f7b37ed0bb54601cde as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/91d8a240b2a944f7b37ed0bb54601cde 2024-11-07T09:37:16,507 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/91d8a240b2a944f7b37ed0bb54601cde, entries=150, sequenceid=395, filesize=12.0 K 2024-11-07T09:37:16,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/e85f51f60e7f468b8ff0449c2594e95f as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/e85f51f60e7f468b8ff0449c2594e95f 2024-11-07T09:37:16,511 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/e85f51f60e7f468b8ff0449c2594e95f, entries=150, sequenceid=395, filesize=12.0 K 2024-11-07T09:37:16,512 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 5165da7af32dbdfd119fd1fa7507f91f in 487ms, sequenceid=395, compaction requested=true 2024-11-07T09:37:16,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:16,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5165da7af32dbdfd119fd1fa7507f91f:A, priority=-2147483648, current under compaction store size is 2 2024-11-07T09:37:16,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:16,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5165da7af32dbdfd119fd1fa7507f91f:B, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:37:16,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:37:16,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5165da7af32dbdfd119fd1fa7507f91f:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:37:16,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T09:37:16,512 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:37:16,513 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:37:16,514 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 5165da7af32dbdfd119fd1fa7507f91f/A is initiating minor compaction (all files) 2024-11-07T09:37:16,514 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5165da7af32dbdfd119fd1fa7507f91f/A in TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:16,514 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/094833d6b09d47cda6cfc4efc49f9ec1, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/80d2176ea5f943ca96e7d4255f4b3b95, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/1d6bb8ebbcb1440ca49c1687b38099b4] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp, totalSize=39.3 K 2024-11-07T09:37:16,515 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 094833d6b09d47cda6cfc4efc49f9ec1, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1730972235116 2024-11-07T09:37:16,515 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80d2176ea5f943ca96e7d4255f4b3b95, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1730972235287 2024-11-07T09:37:16,516 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d6bb8ebbcb1440ca49c1687b38099b4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1730972235974 2024-11-07T09:37:16,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-07T09:37:16,530 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5165da7af32dbdfd119fd1fa7507f91f#A#compaction#294 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:37:16,531 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/d3855aea80e249309d38cba5e145765a is 50, key is test_row_0/A:col10/1730972236022/Put/seqid=0 2024-11-07T09:37:16,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742165_1341 (size=13255) 2024-11-07T09:37:16,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-07T09:37:16,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:16,570 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2902): Flushing 5165da7af32dbdfd119fd1fa7507f91f 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-07T09:37:16,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/ad0ef88c8c984215be41618805b16b45 is 50, key is test_row_0/A:col10/1730972236164/Put/seqid=0 2024-11-07T09:37:16,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742166_1342 (size=12301) 2024-11-07T09:37:16,646 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/afdb7b58ec924e8caa3f813861afe2ad as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/afdb7b58ec924e8caa3f813861afe2ad 2024-11-07T09:37:16,651 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5165da7af32dbdfd119fd1fa7507f91f/C of 5165da7af32dbdfd119fd1fa7507f91f into afdb7b58ec924e8caa3f813861afe2ad(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:37:16,651 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:16,651 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., storeName=5165da7af32dbdfd119fd1fa7507f91f/C, priority=12, startTime=1730972235751; duration=0sec 2024-11-07T09:37:16,651 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T09:37:16,651 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5165da7af32dbdfd119fd1fa7507f91f:C 2024-11-07T09:37:16,651 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5165da7af32dbdfd119fd1fa7507f91f:C 2024-11-07T09:37:16,651 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:37:16,652 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:37:16,652 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 5165da7af32dbdfd119fd1fa7507f91f/B is initiating minor compaction (all files) 2024-11-07T09:37:16,652 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5165da7af32dbdfd119fd1fa7507f91f/B in TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:16,652 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/c64b40607a12478fa446ce8a13076a03, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/652283d14dba47c5ab4dfe832a3d8a03, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/91d8a240b2a944f7b37ed0bb54601cde] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp, totalSize=36.9 K 2024-11-07T09:37:16,653 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting c64b40607a12478fa446ce8a13076a03, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1730972235116 2024-11-07T09:37:16,653 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 652283d14dba47c5ab4dfe832a3d8a03, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1730972235287 2024-11-07T09:37:16,653 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 91d8a240b2a944f7b37ed0bb54601cde, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1730972235985 2024-11-07T09:37:16,660 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5165da7af32dbdfd119fd1fa7507f91f#B#compaction#296 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:37:16,660 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/845c5569af8e4db8936c53b9382b0ce1 is 50, key is test_row_0/B:col10/1730972236022/Put/seqid=0 2024-11-07T09:37:16,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742167_1343 (size=13255) 2024-11-07T09:37:16,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-07T09:37:16,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 5165da7af32dbdfd119fd1fa7507f91f 2024-11-07T09:37:16,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. as already flushing 2024-11-07T09:37:16,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:16,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972296842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:16,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:16,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:16,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:16,941 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/d3855aea80e249309d38cba5e145765a as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/d3855aea80e249309d38cba5e145765a 2024-11-07T09:37:16,946 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5165da7af32dbdfd119fd1fa7507f91f/A of 5165da7af32dbdfd119fd1fa7507f91f into d3855aea80e249309d38cba5e145765a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:37:16,946 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:16,946 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., storeName=5165da7af32dbdfd119fd1fa7507f91f/A, priority=13, startTime=1730972236512; duration=0sec 2024-11-07T09:37:16,946 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:37:16,946 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5165da7af32dbdfd119fd1fa7507f91f:A 2024-11-07T09:37:16,946 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-07T09:37:16,946 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-07T09:37:16,947 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-07T09:37:16,947 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. because compaction request was cancelled 2024-11-07T09:37:16,947 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5165da7af32dbdfd119fd1fa7507f91f:C 2024-11-07T09:37:16,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:16,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972296953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:16,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:16,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:16,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:16,978 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/ad0ef88c8c984215be41618805b16b45 2024-11-07T09:37:16,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/4c82f22898974ba499980b7600776486 is 50, key is test_row_0/B:col10/1730972236164/Put/seqid=0 2024-11-07T09:37:16,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742168_1344 (size=12301) 2024-11-07T09:37:17,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-07T09:37:17,080 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/845c5569af8e4db8936c53b9382b0ce1 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/845c5569af8e4db8936c53b9382b0ce1 2024-11-07T09:37:17,085 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5165da7af32dbdfd119fd1fa7507f91f/B of 5165da7af32dbdfd119fd1fa7507f91f into 845c5569af8e4db8936c53b9382b0ce1(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:37:17,085 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:17,086 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., storeName=5165da7af32dbdfd119fd1fa7507f91f/B, priority=13, startTime=1730972236512; duration=0sec 2024-11-07T09:37:17,086 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:17,086 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5165da7af32dbdfd119fd1fa7507f91f:B 2024-11-07T09:37:17,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:17,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972297162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:17,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:17,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:17,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:17,393 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/4c82f22898974ba499980b7600776486 2024-11-07T09:37:17,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/d7c657e5e0aa4f5a9f8518ad878cb8ae is 50, key is test_row_0/C:col10/1730972236164/Put/seqid=0 2024-11-07T09:37:17,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742169_1345 (size=12301) 2024-11-07T09:37:17,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:17,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972297473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:17,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:17,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:17,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:17,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-07T09:37:17,805 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/d7c657e5e0aa4f5a9f8518ad878cb8ae 2024-11-07T09:37:17,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/ad0ef88c8c984215be41618805b16b45 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/ad0ef88c8c984215be41618805b16b45 2024-11-07T09:37:17,813 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/ad0ef88c8c984215be41618805b16b45, entries=150, sequenceid=418, filesize=12.0 K 2024-11-07T09:37:17,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/4c82f22898974ba499980b7600776486 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/4c82f22898974ba499980b7600776486 2024-11-07T09:37:17,818 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/4c82f22898974ba499980b7600776486, entries=150, sequenceid=418, filesize=12.0 K 2024-11-07T09:37:17,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/d7c657e5e0aa4f5a9f8518ad878cb8ae as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/d7c657e5e0aa4f5a9f8518ad878cb8ae 2024-11-07T09:37:17,823 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/d7c657e5e0aa4f5a9f8518ad878cb8ae, entries=150, sequenceid=418, filesize=12.0 K 2024-11-07T09:37:17,824 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 5165da7af32dbdfd119fd1fa7507f91f in 1253ms, sequenceid=418, compaction requested=true 2024-11-07T09:37:17,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2603): Flush status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:17,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:17,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=87 2024-11-07T09:37:17,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4169): Remote procedure done, pid=87 2024-11-07T09:37:17,827 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-07T09:37:17,827 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4090 sec 2024-11-07T09:37:17,829 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees in 1.4130 sec 2024-11-07T09:37:17,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 5165da7af32dbdfd119fd1fa7507f91f 2024-11-07T09:37:17,985 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:37:17,985 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5165da7af32dbdfd119fd1fa7507f91f 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-07T09:37:17,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/e284817689704ca7bf87f56e40c2a074 is 50, key is test_row_0/A:col10/1730972236840/Put/seqid=0 2024-11-07T09:37:17,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742170_1346 (size=12301) 2024-11-07T09:37:18,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:18,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972298065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:18,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:18,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:18,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:18,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:18,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972298173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:18,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:18,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:18,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:18,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:18,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42826 deadline: 1730972298363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:18,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:18,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730972298363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:18,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:18,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1730972298363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:18,366 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 9f1eb259a29b,42895,1730972153970 for 'test_row_2' in 5165da7af32dbdfd119fd1fa7507f91f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T09:37:18,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:18,367 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:18,367 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:18,367 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:18,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42842 deadline: 1730972298363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:18,367 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 9f1eb259a29b,42895,1730972153970 for 'test_row_2' in 5165da7af32dbdfd119fd1fa7507f91f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T09:37:18,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:18,367 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 9f1eb259a29b,42895,1730972153970 for 'test_row_0' in 5165da7af32dbdfd119fd1fa7507f91f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T09:37:18,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:18,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:18,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:18,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:18,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:18,367 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 9f1eb259a29b,42895,1730972153970 for 'test_row_2' in 5165da7af32dbdfd119fd1fa7507f91f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T09:37:18,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:18,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:18,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:18,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:18,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972298382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:18,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:18,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:18,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:18,395 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/e284817689704ca7bf87f56e40c2a074 2024-11-07T09:37:18,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/780be5a3fafd4e59b64ddcd4a13f4578 is 50, key is test_row_0/B:col10/1730972236840/Put/seqid=0 2024-11-07T09:37:18,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742171_1347 (size=12301) 2024-11-07T09:37:18,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-07T09:37:18,563 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:37:18,564 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T09:37:18,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees 2024-11-07T09:37:18,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-07T09:37:18,566 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T09:37:18,566 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T09:37:18,566 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T09:37:18,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-07T09:37:18,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:18,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972298693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:18,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:18,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:18,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:18,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-07T09:37:18,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:18,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. as already flushing 2024-11-07T09:37:18,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:18,720 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:18,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:18,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:18,811 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/780be5a3fafd4e59b64ddcd4a13f4578 2024-11-07T09:37:18,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/77825b31686b4c66a93cfe14cca4349d is 50, key is test_row_0/C:col10/1730972236840/Put/seqid=0 2024-11-07T09:37:18,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742172_1348 (size=12301) 2024-11-07T09:37:18,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-07T09:37:18,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:18,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. as already flushing 2024-11-07T09:37:18,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:18,873 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:18,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:18,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:18,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-07T09:37:19,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-07T09:37:19,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:19,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. as already flushing 2024-11-07T09:37:19,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:19,026 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:19,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:19,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:19,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-07T09:37:19,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:19,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. as already flushing 2024-11-07T09:37:19,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:19,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:19,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:19,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:19,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-07T09:37:19,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:19,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972299203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:19,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:19,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:19,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:19,223 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/77825b31686b4c66a93cfe14cca4349d 2024-11-07T09:37:19,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/e284817689704ca7bf87f56e40c2a074 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/e284817689704ca7bf87f56e40c2a074 2024-11-07T09:37:19,231 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/e284817689704ca7bf87f56e40c2a074, entries=150, sequenceid=437, filesize=12.0 K 2024-11-07T09:37:19,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/780be5a3fafd4e59b64ddcd4a13f4578 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/780be5a3fafd4e59b64ddcd4a13f4578 2024-11-07T09:37:19,236 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/780be5a3fafd4e59b64ddcd4a13f4578, entries=150, sequenceid=437, filesize=12.0 K 2024-11-07T09:37:19,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/77825b31686b4c66a93cfe14cca4349d as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/77825b31686b4c66a93cfe14cca4349d 2024-11-07T09:37:19,240 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/77825b31686b4c66a93cfe14cca4349d, entries=150, sequenceid=437, filesize=12.0 K 2024-11-07T09:37:19,241 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 5165da7af32dbdfd119fd1fa7507f91f in 1256ms, sequenceid=437, compaction requested=true 2024-11-07T09:37:19,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:19,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5165da7af32dbdfd119fd1fa7507f91f:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T09:37:19,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:19,241 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:37:19,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5165da7af32dbdfd119fd1fa7507f91f:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T09:37:19,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:19,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5165da7af32dbdfd119fd1fa7507f91f:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:37:19,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:37:19,241 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:37:19,242 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:37:19,242 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 5165da7af32dbdfd119fd1fa7507f91f/A is initiating minor compaction (all files) 2024-11-07T09:37:19,242 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5165da7af32dbdfd119fd1fa7507f91f/A in TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:19,242 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/d3855aea80e249309d38cba5e145765a, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/ad0ef88c8c984215be41618805b16b45, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/e284817689704ca7bf87f56e40c2a074] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp, totalSize=37.0 K 2024-11-07T09:37:19,243 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:37:19,243 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 5165da7af32dbdfd119fd1fa7507f91f/B is initiating minor compaction (all files) 2024-11-07T09:37:19,243 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5165da7af32dbdfd119fd1fa7507f91f/B in TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:19,243 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/845c5569af8e4db8936c53b9382b0ce1, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/4c82f22898974ba499980b7600776486, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/780be5a3fafd4e59b64ddcd4a13f4578] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp, totalSize=37.0 K 2024-11-07T09:37:19,243 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3855aea80e249309d38cba5e145765a, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1730972235985 2024-11-07T09:37:19,243 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 845c5569af8e4db8936c53b9382b0ce1, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1730972235985 2024-11-07T09:37:19,243 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad0ef88c8c984215be41618805b16b45, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1730972236132 2024-11-07T09:37:19,243 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c82f22898974ba499980b7600776486, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1730972236132 2024-11-07T09:37:19,244 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting e284817689704ca7bf87f56e40c2a074, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1730972236826 2024-11-07T09:37:19,244 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 780be5a3fafd4e59b64ddcd4a13f4578, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1730972236826 2024-11-07T09:37:19,251 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5165da7af32dbdfd119fd1fa7507f91f#B#compaction#302 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:37:19,251 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5165da7af32dbdfd119fd1fa7507f91f#A#compaction#303 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:37:19,252 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/0859d533f25440a99eb09717f8e92ea3 is 50, key is test_row_0/B:col10/1730972236840/Put/seqid=0 2024-11-07T09:37:19,252 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/a5aaf00701d94f73ab1d1db1783e2312 is 50, key is test_row_0/A:col10/1730972236840/Put/seqid=0 2024-11-07T09:37:19,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742173_1349 (size=13357) 2024-11-07T09:37:19,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742174_1350 (size=13357) 2024-11-07T09:37:19,261 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/0859d533f25440a99eb09717f8e92ea3 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/0859d533f25440a99eb09717f8e92ea3 2024-11-07T09:37:19,265 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5165da7af32dbdfd119fd1fa7507f91f/B of 5165da7af32dbdfd119fd1fa7507f91f into 0859d533f25440a99eb09717f8e92ea3(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:37:19,266 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:19,266 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., storeName=5165da7af32dbdfd119fd1fa7507f91f/B, priority=13, startTime=1730972239241; duration=0sec 2024-11-07T09:37:19,266 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:37:19,266 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5165da7af32dbdfd119fd1fa7507f91f:B 2024-11-07T09:37:19,266 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T09:37:19,267 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T09:37:19,267 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 5165da7af32dbdfd119fd1fa7507f91f/C is initiating minor compaction (all files) 2024-11-07T09:37:19,267 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5165da7af32dbdfd119fd1fa7507f91f/C in TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:19,268 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/afdb7b58ec924e8caa3f813861afe2ad, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/e85f51f60e7f468b8ff0449c2594e95f, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/d7c657e5e0aa4f5a9f8518ad878cb8ae, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/77825b31686b4c66a93cfe14cca4349d] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp, totalSize=48.9 K 2024-11-07T09:37:19,268 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting afdb7b58ec924e8caa3f813861afe2ad, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1730972235287 2024-11-07T09:37:19,268 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting e85f51f60e7f468b8ff0449c2594e95f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1730972235985 2024-11-07T09:37:19,268 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting d7c657e5e0aa4f5a9f8518ad878cb8ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1730972236132 2024-11-07T09:37:19,269 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 77825b31686b4c66a93cfe14cca4349d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1730972236826 2024-11-07T09:37:19,276 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5165da7af32dbdfd119fd1fa7507f91f#C#compaction#304 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:37:19,277 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/0f534407b4a4428c9a7a87ccca738aa5 is 50, key is test_row_0/C:col10/1730972236840/Put/seqid=0 2024-11-07T09:37:19,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742175_1351 (size=13323) 2024-11-07T09:37:19,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-07T09:37:19,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:19,332 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2902): Flushing 5165da7af32dbdfd119fd1fa7507f91f 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-07T09:37:19,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/e17f20c1f1dc4a25bbc474b9686a5c1f is 50, key is test_row_0/A:col10/1730972238058/Put/seqid=0 2024-11-07T09:37:19,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742176_1352 (size=12301) 2024-11-07T09:37:19,662 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/a5aaf00701d94f73ab1d1db1783e2312 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/a5aaf00701d94f73ab1d1db1783e2312 2024-11-07T09:37:19,667 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5165da7af32dbdfd119fd1fa7507f91f/A of 5165da7af32dbdfd119fd1fa7507f91f into a5aaf00701d94f73ab1d1db1783e2312(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:37:19,668 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:19,668 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., storeName=5165da7af32dbdfd119fd1fa7507f91f/A, priority=13, startTime=1730972239241; duration=0sec 2024-11-07T09:37:19,668 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:19,668 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5165da7af32dbdfd119fd1fa7507f91f:A 2024-11-07T09:37:19,690 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/0f534407b4a4428c9a7a87ccca738aa5 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/0f534407b4a4428c9a7a87ccca738aa5 2024-11-07T09:37:19,695 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5165da7af32dbdfd119fd1fa7507f91f/C of 5165da7af32dbdfd119fd1fa7507f91f into 0f534407b4a4428c9a7a87ccca738aa5(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:37:19,695 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:19,695 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., storeName=5165da7af32dbdfd119fd1fa7507f91f/C, priority=12, startTime=1730972239241; duration=0sec 2024-11-07T09:37:19,695 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:19,695 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5165da7af32dbdfd119fd1fa7507f91f:C 2024-11-07T09:37:19,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-07T09:37:19,745 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/e17f20c1f1dc4a25bbc474b9686a5c1f 2024-11-07T09:37:19,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/afce29e21c484b56a0fd0fd8b0d5120e is 50, key is test_row_0/B:col10/1730972238058/Put/seqid=0 2024-11-07T09:37:19,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742177_1353 (size=12301) 2024-11-07T09:37:20,162 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/afce29e21c484b56a0fd0fd8b0d5120e 2024-11-07T09:37:20,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/70e41c07fb6d42f983d80c662d65aa2d is 50, key is test_row_0/C:col10/1730972238058/Put/seqid=0 2024-11-07T09:37:20,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742178_1354 (size=12301) 2024-11-07T09:37:20,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 5165da7af32dbdfd119fd1fa7507f91f 2024-11-07T09:37:20,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. as already flushing 2024-11-07T09:37:20,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:20,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972300281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:20,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:20,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:20,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:20,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:20,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972300393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:20,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:20,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:20,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:20,576 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/70e41c07fb6d42f983d80c662d65aa2d 2024-11-07T09:37:20,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/e17f20c1f1dc4a25bbc474b9686a5c1f as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/e17f20c1f1dc4a25bbc474b9686a5c1f 2024-11-07T09:37:20,584 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/e17f20c1f1dc4a25bbc474b9686a5c1f, entries=150, sequenceid=458, filesize=12.0 K 2024-11-07T09:37:20,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/afce29e21c484b56a0fd0fd8b0d5120e as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/afce29e21c484b56a0fd0fd8b0d5120e 2024-11-07T09:37:20,589 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/afce29e21c484b56a0fd0fd8b0d5120e, entries=150, sequenceid=458, filesize=12.0 K 2024-11-07T09:37:20,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/70e41c07fb6d42f983d80c662d65aa2d as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/70e41c07fb6d42f983d80c662d65aa2d 2024-11-07T09:37:20,593 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/70e41c07fb6d42f983d80c662d65aa2d, entries=150, sequenceid=458, filesize=12.0 K 2024-11-07T09:37:20,594 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 5165da7af32dbdfd119fd1fa7507f91f in 1262ms, sequenceid=458, compaction requested=false 2024-11-07T09:37:20,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2603): Flush status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:20,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:20,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=89 2024-11-07T09:37:20,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4169): Remote procedure done, pid=89 2024-11-07T09:37:20,597 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-07T09:37:20,597 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0290 sec 2024-11-07T09:37:20,599 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees in 2.0340 sec 2024-11-07T09:37:20,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 5165da7af32dbdfd119fd1fa7507f91f 2024-11-07T09:37:20,606 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:37:20,606 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5165da7af32dbdfd119fd1fa7507f91f 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-07T09:37:20,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/e28d60f8f55c45dabcb8b7174ebe057b is 50, key is test_row_0/A:col10/1730972240605/Put/seqid=0 2024-11-07T09:37:20,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742179_1355 (size=14741) 2024-11-07T09:37:20,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:20,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972300685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:20,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:20,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:20,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:20,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-07T09:37:20,714 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:37:20,714 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T09:37:20,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees 2024-11-07T09:37:20,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-07T09:37:20,716 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=90, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T09:37:20,717 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=90, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T09:37:20,717 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T09:37:20,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:20,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972300793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:20,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:20,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:20,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:20,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-07T09:37:20,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-07T09:37:20,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:20,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. as already flushing 2024-11-07T09:37:20,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:20,869 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:20,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:20,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:21,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:21,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972301003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:21,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:21,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:21,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:21,016 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/e28d60f8f55c45dabcb8b7174ebe057b 2024-11-07T09:37:21,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-07T09:37:21,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:21,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. as already flushing 2024-11-07T09:37:21,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:21,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:21,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:21,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:21,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/83384e9097ee4352817ad2a4654839ec is 50, key is test_row_0/B:col10/1730972240605/Put/seqid=0 2024-11-07T09:37:21,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742180_1356 (size=12301) 2024-11-07T09:37:21,030 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/83384e9097ee4352817ad2a4654839ec 2024-11-07T09:37:21,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-07T09:37:21,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/918b911c25d44771886a01f965bb7dce is 50, key is test_row_0/C:col10/1730972240605/Put/seqid=0 2024-11-07T09:37:21,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742181_1357 (size=12301) 2024-11-07T09:37:21,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-07T09:37:21,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:21,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. as already flushing 2024-11-07T09:37:21,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:21,175 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:21,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:21,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:21,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:21,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972301313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:21,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:21,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:21,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:21,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-07T09:37:21,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:21,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. as already flushing 2024-11-07T09:37:21,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:21,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:21,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:21,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:21,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-07T09:37:21,442 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/918b911c25d44771886a01f965bb7dce 2024-11-07T09:37:21,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/e28d60f8f55c45dabcb8b7174ebe057b as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/e28d60f8f55c45dabcb8b7174ebe057b 2024-11-07T09:37:21,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/e28d60f8f55c45dabcb8b7174ebe057b, entries=200, sequenceid=477, filesize=14.4 K 2024-11-07T09:37:21,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/83384e9097ee4352817ad2a4654839ec as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/83384e9097ee4352817ad2a4654839ec 2024-11-07T09:37:21,456 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/83384e9097ee4352817ad2a4654839ec, entries=150, sequenceid=477, filesize=12.0 K 2024-11-07T09:37:21,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/918b911c25d44771886a01f965bb7dce as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/918b911c25d44771886a01f965bb7dce 2024-11-07T09:37:21,460 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/918b911c25d44771886a01f965bb7dce, entries=150, sequenceid=477, filesize=12.0 K 2024-11-07T09:37:21,461 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 5165da7af32dbdfd119fd1fa7507f91f in 855ms, sequenceid=477, compaction requested=true 2024-11-07T09:37:21,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:21,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5165da7af32dbdfd119fd1fa7507f91f:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T09:37:21,461 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:37:21,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:21,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5165da7af32dbdfd119fd1fa7507f91f:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T09:37:21,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:21,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5165da7af32dbdfd119fd1fa7507f91f:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:37:21,461 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:37:21,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:37:21,462 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40399 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:37:21,462 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:37:21,462 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 5165da7af32dbdfd119fd1fa7507f91f/A is initiating minor compaction (all files) 2024-11-07T09:37:21,462 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 5165da7af32dbdfd119fd1fa7507f91f/B is initiating minor compaction (all files) 2024-11-07T09:37:21,462 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5165da7af32dbdfd119fd1fa7507f91f/A in TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:21,462 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5165da7af32dbdfd119fd1fa7507f91f/B in TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:21,462 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/a5aaf00701d94f73ab1d1db1783e2312, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/e17f20c1f1dc4a25bbc474b9686a5c1f, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/e28d60f8f55c45dabcb8b7174ebe057b] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp, totalSize=39.5 K 2024-11-07T09:37:21,462 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/0859d533f25440a99eb09717f8e92ea3, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/afce29e21c484b56a0fd0fd8b0d5120e, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/83384e9097ee4352817ad2a4654839ec] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp, totalSize=37.1 K 2024-11-07T09:37:21,463 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5aaf00701d94f73ab1d1db1783e2312, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1730972236826 2024-11-07T09:37:21,463 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 0859d533f25440a99eb09717f8e92ea3, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1730972236826 2024-11-07T09:37:21,463 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting e17f20c1f1dc4a25bbc474b9686a5c1f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1730972238052 2024-11-07T09:37:21,463 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting afce29e21c484b56a0fd0fd8b0d5120e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1730972238052 2024-11-07T09:37:21,463 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting e28d60f8f55c45dabcb8b7174ebe057b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1730972240263 2024-11-07T09:37:21,464 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 83384e9097ee4352817ad2a4654839ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1730972240269 2024-11-07T09:37:21,478 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5165da7af32dbdfd119fd1fa7507f91f#A#compaction#311 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:37:21,479 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/4f7c1643bb5b4f8e9ddda5cd84f62ae9 is 50, key is test_row_0/A:col10/1730972240605/Put/seqid=0 2024-11-07T09:37:21,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-07T09:37:21,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:21,482 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2902): Flushing 5165da7af32dbdfd119fd1fa7507f91f 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-07T09:37:21,498 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5165da7af32dbdfd119fd1fa7507f91f#B#compaction#312 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:37:21,498 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/0bcebb9ede614f1b8962b4c30cd14fb8 is 50, key is test_row_0/B:col10/1730972240605/Put/seqid=0 2024-11-07T09:37:21,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/e7dd334c3de84252a30cbd65d129893e is 50, key is test_row_0/A:col10/1730972240672/Put/seqid=0 2024-11-07T09:37:21,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742182_1358 (size=13459) 2024-11-07T09:37:21,556 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/4f7c1643bb5b4f8e9ddda5cd84f62ae9 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/4f7c1643bb5b4f8e9ddda5cd84f62ae9 2024-11-07T09:37:21,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742183_1359 (size=13459) 2024-11-07T09:37:21,563 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5165da7af32dbdfd119fd1fa7507f91f/A of 5165da7af32dbdfd119fd1fa7507f91f into 4f7c1643bb5b4f8e9ddda5cd84f62ae9(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:37:21,563 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:21,563 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., storeName=5165da7af32dbdfd119fd1fa7507f91f/A, priority=13, startTime=1730972241461; duration=0sec 2024-11-07T09:37:21,563 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:37:21,564 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5165da7af32dbdfd119fd1fa7507f91f:A 2024-11-07T09:37:21,564 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:37:21,566 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:37:21,566 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 5165da7af32dbdfd119fd1fa7507f91f/C is initiating minor compaction (all files) 2024-11-07T09:37:21,567 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5165da7af32dbdfd119fd1fa7507f91f/C in TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:21,567 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/0f534407b4a4428c9a7a87ccca738aa5, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/70e41c07fb6d42f983d80c662d65aa2d, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/918b911c25d44771886a01f965bb7dce] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp, totalSize=37.0 K 2024-11-07T09:37:21,567 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f534407b4a4428c9a7a87ccca738aa5, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1730972236826 2024-11-07T09:37:21,567 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/0bcebb9ede614f1b8962b4c30cd14fb8 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/0bcebb9ede614f1b8962b4c30cd14fb8 2024-11-07T09:37:21,568 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70e41c07fb6d42f983d80c662d65aa2d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1730972238052 2024-11-07T09:37:21,568 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 918b911c25d44771886a01f965bb7dce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1730972240269 2024-11-07T09:37:21,575 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5165da7af32dbdfd119fd1fa7507f91f/B of 5165da7af32dbdfd119fd1fa7507f91f into 0bcebb9ede614f1b8962b4c30cd14fb8(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:37:21,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742184_1360 (size=12301) 2024-11-07T09:37:21,575 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:21,575 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., storeName=5165da7af32dbdfd119fd1fa7507f91f/B, priority=13, startTime=1730972241461; duration=0sec 2024-11-07T09:37:21,575 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:21,575 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5165da7af32dbdfd119fd1fa7507f91f:B 2024-11-07T09:37:21,576 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/e7dd334c3de84252a30cbd65d129893e 2024-11-07T09:37:21,583 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5165da7af32dbdfd119fd1fa7507f91f#C#compaction#314 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:37:21,584 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/8a453281d6ea47dea1b1d1144119cd68 is 50, key is test_row_0/C:col10/1730972240605/Put/seqid=0 2024-11-07T09:37:21,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/f314ccb21e1942db9fa039fb7f60a4f8 is 50, key is test_row_0/B:col10/1730972240672/Put/seqid=0 2024-11-07T09:37:21,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742185_1361 (size=13425) 2024-11-07T09:37:21,601 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/8a453281d6ea47dea1b1d1144119cd68 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/8a453281d6ea47dea1b1d1144119cd68 2024-11-07T09:37:21,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742186_1362 (size=12301) 2024-11-07T09:37:21,608 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5165da7af32dbdfd119fd1fa7507f91f/C of 5165da7af32dbdfd119fd1fa7507f91f into 8a453281d6ea47dea1b1d1144119cd68(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:37:21,608 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:21,608 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., storeName=5165da7af32dbdfd119fd1fa7507f91f/C, priority=13, startTime=1730972241461; duration=0sec 2024-11-07T09:37:21,608 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:21,608 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5165da7af32dbdfd119fd1fa7507f91f:C 2024-11-07T09:37:21,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 5165da7af32dbdfd119fd1fa7507f91f 2024-11-07T09:37:21,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. as already flushing 2024-11-07T09:37:21,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-07T09:37:21,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:21,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972301883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:21,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:21,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:21,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:21,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:21,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972301993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:21,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:21,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:21,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:22,006 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/f314ccb21e1942db9fa039fb7f60a4f8 2024-11-07T09:37:22,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/989a3731eadf480a98d39ffe9a8734c3 is 50, key is test_row_0/C:col10/1730972240672/Put/seqid=0 2024-11-07T09:37:22,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742187_1363 (size=12301) 2024-11-07T09:37:22,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:22,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972302203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:22,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:22,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:22,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:22,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:22,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42826 deadline: 1730972302383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:22,386 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 9f1eb259a29b,42895,1730972153970 for 'test_row_2' in 5165da7af32dbdfd119fd1fa7507f91f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T09:37:22,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:22,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:22,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:22,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:22,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1730972302392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:22,395 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 9f1eb259a29b,42895,1730972153970 for 'test_row_0' in 5165da7af32dbdfd119fd1fa7507f91f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T09:37:22,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:22,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:22,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:22,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:22,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42842 deadline: 1730972302403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:22,406 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 9f1eb259a29b,42895,1730972153970 for 'test_row_2' in 5165da7af32dbdfd119fd1fa7507f91f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T09:37:22,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:22,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:22,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:22,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:22,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42856 deadline: 1730972302413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:22,416 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 9f1eb259a29b,42895,1730972153970 for 'test_row_2' in 5165da7af32dbdfd119fd1fa7507f91f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8231 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T09:37:22,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:22,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:22,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:22,419 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/989a3731eadf480a98d39ffe9a8734c3 2024-11-07T09:37:22,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/e7dd334c3de84252a30cbd65d129893e as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/e7dd334c3de84252a30cbd65d129893e 2024-11-07T09:37:22,428 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/e7dd334c3de84252a30cbd65d129893e, entries=150, sequenceid=496, filesize=12.0 K 2024-11-07T09:37:22,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/f314ccb21e1942db9fa039fb7f60a4f8 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/f314ccb21e1942db9fa039fb7f60a4f8 2024-11-07T09:37:22,432 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/f314ccb21e1942db9fa039fb7f60a4f8, entries=150, sequenceid=496, filesize=12.0 K 2024-11-07T09:37:22,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/989a3731eadf480a98d39ffe9a8734c3 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/989a3731eadf480a98d39ffe9a8734c3 2024-11-07T09:37:22,438 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/989a3731eadf480a98d39ffe9a8734c3, entries=150, sequenceid=496, filesize=12.0 K 2024-11-07T09:37:22,440 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 5165da7af32dbdfd119fd1fa7507f91f in 958ms, sequenceid=496, compaction requested=false 2024-11-07T09:37:22,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2603): Flush status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:22,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:22,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=91 2024-11-07T09:37:22,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4169): Remote procedure done, pid=91 2024-11-07T09:37:22,442 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T09:37:22,444 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-07T09:37:22,444 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7250 sec 2024-11-07T09:37:22,445 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees in 1.7300 sec 2024-11-07T09:37:22,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 5165da7af32dbdfd119fd1fa7507f91f 2024-11-07T09:37:22,516 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:37:22,516 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5165da7af32dbdfd119fd1fa7507f91f 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-07T09:37:22,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/39a850f29a224a7fb2badcaaf5cff9b9 is 50, key is test_row_0/A:col10/1730972241867/Put/seqid=0 2024-11-07T09:37:22,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742188_1364 (size=14741) 2024-11-07T09:37:22,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:22,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 269 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972302589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:22,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:22,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:22,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:22,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:22,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 270 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972302703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:22,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:22,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:22,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:22,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-07T09:37:22,863 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:37:22,864 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T09:37:22,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=92, table=TestAcidGuarantees 2024-11-07T09:37:22,865 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=92, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=92, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T09:37:22,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-07T09:37:22,866 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=92, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=92, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T09:37:22,866 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T09:37:22,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:22,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 271 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972302913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:22,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:22,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:22,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:22,945 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=517 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/39a850f29a224a7fb2badcaaf5cff9b9 2024-11-07T09:37:22,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/4568c06e5c134c3ba14e26b719e7fe66 is 50, key is test_row_0/B:col10/1730972241867/Put/seqid=0 2024-11-07T09:37:22,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742189_1365 (size=12301) 2024-11-07T09:37:22,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-07T09:37:23,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=93 2024-11-07T09:37:23,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:23,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. as already flushing 2024-11-07T09:37:23,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:23,019 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] handler.RSProcedureHandler(58): pid=93 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:23,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=93 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:23,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=93 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:23,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=93 2024-11-07T09:37:23,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:23,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. as already flushing 2024-11-07T09:37:23,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:23,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] handler.RSProcedureHandler(58): pid=93 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:23,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=93 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:23,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=93 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:23,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-07T09:37:23,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:23,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 272 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972303223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:23,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:23,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:23,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:23,252 INFO [Thread-1195 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1195. 2024-11-07T09:37:23,252 INFO [Thread-1199 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1199. 2024-11-07T09:37:23,252 DEBUG [Thread-1195 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:37:23,252 DEBUG [Thread-1199 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:37:23,252 DEBUG [Thread-1195 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:23,252 DEBUG [Thread-1199 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:23,252 DEBUG [Thread-1195 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:23,252 DEBUG [Thread-1199 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:23,252 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:37:23,252 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:37:23,256 INFO [Thread-1197 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1197. 2024-11-07T09:37:23,256 INFO [Thread-1193 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1193. 2024-11-07T09:37:23,256 INFO [Thread-1201 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1201. 2024-11-07T09:37:23,256 DEBUG [Thread-1197 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:37:23,256 DEBUG [Thread-1201 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:37:23,256 DEBUG [Thread-1193 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:37:23,256 DEBUG [Thread-1197 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:23,256 DEBUG [Thread-1201 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:23,256 DEBUG [Thread-1193 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:23,257 DEBUG [Thread-1193 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:23,257 DEBUG [Thread-1201 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:23,257 DEBUG [Thread-1197 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:23,257 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:37:23,257 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:37:23,257 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:37:23,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=93 2024-11-07T09:37:23,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:23,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. as already flushing 2024-11-07T09:37:23,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:23,325 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] handler.RSProcedureHandler(58): pid=93 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:23,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=93 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:23,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=93 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:23,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=517 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/4568c06e5c134c3ba14e26b719e7fe66 2024-11-07T09:37:23,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/5b8246936e5c455bab712c30000491fe is 50, key is test_row_0/C:col10/1730972241867/Put/seqid=0 2024-11-07T09:37:23,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742190_1366 (size=12301) 2024-11-07T09:37:23,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=93 2024-11-07T09:37:23,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:23,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. as already flushing 2024-11-07T09:37:23,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:23,478 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] handler.RSProcedureHandler(58): pid=93 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:23,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=93 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:23,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=93 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:23,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-07T09:37:23,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=93 2024-11-07T09:37:23,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:23,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. as already flushing 2024-11-07T09:37:23,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:23,631 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] handler.RSProcedureHandler(58): pid=93 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:23,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=93 java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:23,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=93 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:23,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:23,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 273 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42878 deadline: 1730972303733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:23,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:23,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:23,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:37:23,767 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=517 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/5b8246936e5c455bab712c30000491fe 2024-11-07T09:37:23,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/39a850f29a224a7fb2badcaaf5cff9b9 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/39a850f29a224a7fb2badcaaf5cff9b9 2024-11-07T09:37:23,774 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/39a850f29a224a7fb2badcaaf5cff9b9, entries=200, sequenceid=517, filesize=14.4 K 2024-11-07T09:37:23,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/4568c06e5c134c3ba14e26b719e7fe66 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/4568c06e5c134c3ba14e26b719e7fe66 2024-11-07T09:37:23,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/4568c06e5c134c3ba14e26b719e7fe66, entries=150, sequenceid=517, filesize=12.0 K 2024-11-07T09:37:23,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/5b8246936e5c455bab712c30000491fe as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/5b8246936e5c455bab712c30000491fe 2024-11-07T09:37:23,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/5b8246936e5c455bab712c30000491fe, entries=150, sequenceid=517, filesize=12.0 K 2024-11-07T09:37:23,781 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 5165da7af32dbdfd119fd1fa7507f91f in 1265ms, sequenceid=517, compaction requested=true 2024-11-07T09:37:23,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:23,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5165da7af32dbdfd119fd1fa7507f91f:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T09:37:23,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:23,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5165da7af32dbdfd119fd1fa7507f91f:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T09:37:23,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:23,782 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:37:23,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5165da7af32dbdfd119fd1fa7507f91f:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:37:23,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:37:23,782 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:37:23,782 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40501 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:37:23,782 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:37:23,782 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 5165da7af32dbdfd119fd1fa7507f91f/A is initiating minor compaction (all files) 2024-11-07T09:37:23,782 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 5165da7af32dbdfd119fd1fa7507f91f/B is initiating minor compaction (all files) 2024-11-07T09:37:23,782 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5165da7af32dbdfd119fd1fa7507f91f/A in TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:23,782 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5165da7af32dbdfd119fd1fa7507f91f/B in TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:23,783 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/4f7c1643bb5b4f8e9ddda5cd84f62ae9, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/e7dd334c3de84252a30cbd65d129893e, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/39a850f29a224a7fb2badcaaf5cff9b9] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp, totalSize=39.6 K 2024-11-07T09:37:23,783 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/0bcebb9ede614f1b8962b4c30cd14fb8, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/f314ccb21e1942db9fa039fb7f60a4f8, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/4568c06e5c134c3ba14e26b719e7fe66] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp, totalSize=37.2 K 2024-11-07T09:37:23,783 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f7c1643bb5b4f8e9ddda5cd84f62ae9, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1730972240269 2024-11-07T09:37:23,783 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bcebb9ede614f1b8962b4c30cd14fb8, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1730972240269 2024-11-07T09:37:23,783 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7dd334c3de84252a30cbd65d129893e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1730972240672 2024-11-07T09:37:23,783 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting f314ccb21e1942db9fa039fb7f60a4f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1730972240672 2024-11-07T09:37:23,783 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39a850f29a224a7fb2badcaaf5cff9b9, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=517, earliestPutTs=1730972241867 2024-11-07T09:37:23,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=93 2024-11-07T09:37:23,783 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 4568c06e5c134c3ba14e26b719e7fe66, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=517, earliestPutTs=1730972241867 2024-11-07T09:37:23,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:23,784 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegion(2902): Flushing 5165da7af32dbdfd119fd1fa7507f91f 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-07T09:37:23,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/602491f7d1ed41b0a7b5d7c0be59d770 is 50, key is test_row_0/A:col10/1730972242587/Put/seqid=0 2024-11-07T09:37:23,789 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5165da7af32dbdfd119fd1fa7507f91f#A#compaction#321 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:37:23,789 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5165da7af32dbdfd119fd1fa7507f91f#B#compaction#322 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:37:23,789 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/490aeacd52cd4ba883acf2b949f0d0e5 is 50, key is test_row_0/A:col10/1730972241867/Put/seqid=0 2024-11-07T09:37:23,790 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/909333a2aea641d585f2570e225e4346 is 50, key is test_row_0/B:col10/1730972241867/Put/seqid=0 2024-11-07T09:37:23,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742191_1367 (size=12301) 2024-11-07T09:37:23,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742192_1368 (size=13561) 2024-11-07T09:37:23,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742193_1369 (size=13561) 2024-11-07T09:37:24,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-07T09:37:24,192 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=535 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/602491f7d1ed41b0a7b5d7c0be59d770 2024-11-07T09:37:24,197 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/909333a2aea641d585f2570e225e4346 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/909333a2aea641d585f2570e225e4346 2024-11-07T09:37:24,197 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/490aeacd52cd4ba883acf2b949f0d0e5 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/490aeacd52cd4ba883acf2b949f0d0e5 2024-11-07T09:37:24,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/11c6b92ab3e2469cb0fbc2b6da8a782c is 50, key is test_row_0/B:col10/1730972242587/Put/seqid=0 2024-11-07T09:37:24,201 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5165da7af32dbdfd119fd1fa7507f91f/B of 5165da7af32dbdfd119fd1fa7507f91f into 909333a2aea641d585f2570e225e4346(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:37:24,201 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5165da7af32dbdfd119fd1fa7507f91f/A of 5165da7af32dbdfd119fd1fa7507f91f into 490aeacd52cd4ba883acf2b949f0d0e5(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:37:24,201 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:24,201 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:24,201 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., storeName=5165da7af32dbdfd119fd1fa7507f91f/B, priority=13, startTime=1730972243782; duration=0sec 2024-11-07T09:37:24,201 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., storeName=5165da7af32dbdfd119fd1fa7507f91f/A, priority=13, startTime=1730972243782; duration=0sec 2024-11-07T09:37:24,201 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:37:24,201 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5165da7af32dbdfd119fd1fa7507f91f:B 2024-11-07T09:37:24,201 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:24,201 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5165da7af32dbdfd119fd1fa7507f91f:A 2024-11-07T09:37:24,201 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:37:24,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742194_1370 (size=12301) 2024-11-07T09:37:24,202 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=535 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/11c6b92ab3e2469cb0fbc2b6da8a782c 2024-11-07T09:37:24,202 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:37:24,202 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 5165da7af32dbdfd119fd1fa7507f91f/C is initiating minor compaction (all files) 2024-11-07T09:37:24,202 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5165da7af32dbdfd119fd1fa7507f91f/C in TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:24,202 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/8a453281d6ea47dea1b1d1144119cd68, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/989a3731eadf480a98d39ffe9a8734c3, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/5b8246936e5c455bab712c30000491fe] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp, totalSize=37.1 K 2024-11-07T09:37:24,203 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a453281d6ea47dea1b1d1144119cd68, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1730972240269 2024-11-07T09:37:24,203 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 989a3731eadf480a98d39ffe9a8734c3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1730972240672 2024-11-07T09:37:24,203 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b8246936e5c455bab712c30000491fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=517, earliestPutTs=1730972241867 2024-11-07T09:37:24,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/62a53f48b86249d8b590157cd5028eb1 is 50, key is test_row_0/C:col10/1730972242587/Put/seqid=0 2024-11-07T09:37:24,209 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5165da7af32dbdfd119fd1fa7507f91f#C#compaction#325 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:37:24,209 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/8abd6e0fe1fc497e845704eebf688776 is 50, key is test_row_0/C:col10/1730972241867/Put/seqid=0 2024-11-07T09:37:24,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742195_1371 (size=12301) 2024-11-07T09:37:24,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742196_1372 (size=13527) 2024-11-07T09:37:24,217 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/8abd6e0fe1fc497e845704eebf688776 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/8abd6e0fe1fc497e845704eebf688776 2024-11-07T09:37:24,220 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5165da7af32dbdfd119fd1fa7507f91f/C of 5165da7af32dbdfd119fd1fa7507f91f into 8abd6e0fe1fc497e845704eebf688776(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:37:24,221 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:24,221 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., storeName=5165da7af32dbdfd119fd1fa7507f91f/C, priority=13, startTime=1730972243782; duration=0sec 2024-11-07T09:37:24,221 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:24,221 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5165da7af32dbdfd119fd1fa7507f91f:C 2024-11-07T09:37:24,611 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=535 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/62a53f48b86249d8b590157cd5028eb1 2024-11-07T09:37:24,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/602491f7d1ed41b0a7b5d7c0be59d770 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/602491f7d1ed41b0a7b5d7c0be59d770 2024-11-07T09:37:24,617 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/602491f7d1ed41b0a7b5d7c0be59d770, entries=150, sequenceid=535, filesize=12.0 K 2024-11-07T09:37:24,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/11c6b92ab3e2469cb0fbc2b6da8a782c as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/11c6b92ab3e2469cb0fbc2b6da8a782c 2024-11-07T09:37:24,621 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/11c6b92ab3e2469cb0fbc2b6da8a782c, entries=150, sequenceid=535, filesize=12.0 K 2024-11-07T09:37:24,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/62a53f48b86249d8b590157cd5028eb1 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/62a53f48b86249d8b590157cd5028eb1 2024-11-07T09:37:24,624 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/62a53f48b86249d8b590157cd5028eb1, entries=150, sequenceid=535, filesize=12.0 K 2024-11-07T09:37:24,625 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=0 B/0 for 5165da7af32dbdfd119fd1fa7507f91f in 841ms, sequenceid=535, compaction requested=false 2024-11-07T09:37:24,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegion(2603): Flush status journal for 5165da7af32dbdfd119fd1fa7507f91f: 2024-11-07T09:37:24,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:24,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=93 2024-11-07T09:37:24,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4169): Remote procedure done, pid=93 2024-11-07T09:37:24,627 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-07T09:37:24,627 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7600 sec 2024-11-07T09:37:24,628 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=92, table=TestAcidGuarantees in 1.7640 sec 2024-11-07T09:37:24,754 INFO [Thread-1182 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1182. 2024-11-07T09:37:24,754 DEBUG [Thread-1182 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:37:24,754 DEBUG [Thread-1182 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:24,754 DEBUG [Thread-1182 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:24,755 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:37:25,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-07T09:37:25,013 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:37:32,444 INFO [Thread-1190 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1190. 2024-11-07T09:37:32,444 DEBUG [Thread-1190 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:37:32,445 DEBUG [Thread-1190 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:32,445 DEBUG [Thread-1190 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:32,445 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:37:32,465 INFO [Thread-1186 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1186. 2024-11-07T09:37:32,465 DEBUG [Thread-1186 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:37:32,465 DEBUG [Thread-1186 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:32,465 DEBUG [Thread-1186 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:32,465 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:37:32,474 INFO [Thread-1184 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1184. 2024-11-07T09:37:32,474 DEBUG [Thread-1184 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:37:32,474 DEBUG [Thread-1184 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:32,474 DEBUG [Thread-1184 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:32,474 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:37:32,504 INFO [Thread-1188 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1188. 2024-11-07T09:37:32,504 DEBUG [Thread-1188 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:37:32,504 DEBUG [Thread-1188 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:32,504 DEBUG [Thread-1188 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:32,504 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:37:32,505 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-07T09:37:32,505 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 207 2024-11-07T09:37:32,505 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-07T09:37:32,505 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-11-07T09:37:32,505 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-07T09:37:32,505 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-07T09:37:32,505 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-07T09:37:32,505 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3257 2024-11-07T09:37:32,505 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3058 2024-11-07T09:37:32,505 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3092 2024-11-07T09:37:32,505 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3253 2024-11-07T09:37:32,505 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3048 2024-11-07T09:37:32,505 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-07T09:37:32,505 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T09:37:32,505 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T09:37:32,505 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:32,505 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:32,505 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:37:32,506 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-07T09:37:32,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=94, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-07T09:37:32,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-07T09:37:32,508 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730972252508"}]},"ts":"1730972252508"} 2024-11-07T09:37:32,509 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-07T09:37:32,509 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-07T09:37:32,510 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T09:37:32,511 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5165da7af32dbdfd119fd1fa7507f91f, UNASSIGN}] 2024-11-07T09:37:32,512 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5165da7af32dbdfd119fd1fa7507f91f, UNASSIGN 2024-11-07T09:37:32,513 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=5165da7af32dbdfd119fd1fa7507f91f, regionState=CLOSING, regionLocation=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:32,514 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5165da7af32dbdfd119fd1fa7507f91f, UNASSIGN because future has completed 2024-11-07T09:37:32,514 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T09:37:32,514 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970}] 2024-11-07T09:37:32,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-07T09:37:32,666 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(122): Close 5165da7af32dbdfd119fd1fa7507f91f 2024-11-07T09:37:32,666 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T09:37:32,666 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1722): Closing 5165da7af32dbdfd119fd1fa7507f91f, disabling compactions & flushes 2024-11-07T09:37:32,666 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:32,666 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:32,666 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. after waiting 0 ms 2024-11-07T09:37:32,666 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:32,667 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(2902): Flushing 5165da7af32dbdfd119fd1fa7507f91f 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-07T09:37:32,670 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/949c163054c04419807fc49fff84677d is 50, key is test_row_0/A:col10/1730972252503/Put/seqid=0 2024-11-07T09:37:32,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742197_1373 (size=9857) 2024-11-07T09:37:32,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-07T09:37:33,075 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=546 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/949c163054c04419807fc49fff84677d 2024-11-07T09:37:33,081 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/b6d61e4dcf9a42e29f44fff0162062e4 is 50, key is test_row_0/B:col10/1730972252503/Put/seqid=0 2024-11-07T09:37:33,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742198_1374 (size=9857) 2024-11-07T09:37:33,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-07T09:37:33,484 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=546 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/b6d61e4dcf9a42e29f44fff0162062e4 2024-11-07T09:37:33,490 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/6cb2809ffdd949f49513edb91de16974 is 50, key is test_row_0/C:col10/1730972252503/Put/seqid=0 2024-11-07T09:37:33,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742199_1375 (size=9857) 2024-11-07T09:37:33,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-07T09:37:33,894 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=546 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/6cb2809ffdd949f49513edb91de16974 2024-11-07T09:37:33,898 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/A/949c163054c04419807fc49fff84677d as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/949c163054c04419807fc49fff84677d 2024-11-07T09:37:33,900 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/949c163054c04419807fc49fff84677d, entries=100, sequenceid=546, filesize=9.6 K 2024-11-07T09:37:33,901 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/B/b6d61e4dcf9a42e29f44fff0162062e4 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/b6d61e4dcf9a42e29f44fff0162062e4 2024-11-07T09:37:33,903 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/b6d61e4dcf9a42e29f44fff0162062e4, entries=100, sequenceid=546, filesize=9.6 K 2024-11-07T09:37:33,904 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/.tmp/C/6cb2809ffdd949f49513edb91de16974 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/6cb2809ffdd949f49513edb91de16974 2024-11-07T09:37:33,907 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/6cb2809ffdd949f49513edb91de16974, entries=100, sequenceid=546, filesize=9.6 K 2024-11-07T09:37:33,907 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 5165da7af32dbdfd119fd1fa7507f91f in 1241ms, sequenceid=546, compaction requested=true 2024-11-07T09:37:33,908 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/f038e89264834b0a8150eac5d1beec01, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/dacc80c57c334f31be4b974b0de66fc6, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/4aa193391d4a4cbe9d33aba3559a39f7, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/2b58c0f555904d7e930f1857a61a7d75, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/f6903116b51345c8babd0eada216f02d, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/45d6fc8e43944071a62278cd0a1e7ebc, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/d75bb0c439da419ebf5558fedbc1e76d, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/30c9caef9cbe416fb69d974b947c95aa, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/78c512af0fe54c6cb5a3115ab9f8bd1e, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/157c6d58e9d8403ca2bf402c42f97646, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/ed34b001e7914b6cb45dcb9627954b1b, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/7ee99ea076304c13accbfced55cf7ab0, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/fdf4034adfa2457e8df7e60dae72f0d2, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/468523535622432f9b9a4760d669b7df, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/eda6886d0f7c46b58175d585d9c8040a, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/d10316279e4b4152944e430b5c2ecc5b, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/c4a909ffe3284dff91e90db6ae06ccad, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/39a8040f3bfe45dd84059e201d5380e6, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/d0a4c7369e1c430db8ea11dd53193400, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/63471011cc884ce4b60054549d803cdb, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/66951a2517e843998a292a4f6ef4f7df, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/3cba0eee27b143a091e8bec2967076da, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/03c58a40343249e7af5596a53b222a21, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/a3e80b4e1f5e4ab3abc24e84c3f54225, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/90e1e75c2a7e40fd81c5af551b6d67c5, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/094833d6b09d47cda6cfc4efc49f9ec1, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/80d2176ea5f943ca96e7d4255f4b3b95, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/1d6bb8ebbcb1440ca49c1687b38099b4, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/d3855aea80e249309d38cba5e145765a, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/ad0ef88c8c984215be41618805b16b45, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/a5aaf00701d94f73ab1d1db1783e2312, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/e284817689704ca7bf87f56e40c2a074, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/e17f20c1f1dc4a25bbc474b9686a5c1f, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/e28d60f8f55c45dabcb8b7174ebe057b, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/4f7c1643bb5b4f8e9ddda5cd84f62ae9, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/e7dd334c3de84252a30cbd65d129893e, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/39a850f29a224a7fb2badcaaf5cff9b9] to archive 2024-11-07T09:37:33,909 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T09:37:33,910 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/f038e89264834b0a8150eac5d1beec01 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/f038e89264834b0a8150eac5d1beec01 2024-11-07T09:37:33,911 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/dacc80c57c334f31be4b974b0de66fc6 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/dacc80c57c334f31be4b974b0de66fc6 2024-11-07T09:37:33,912 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/4aa193391d4a4cbe9d33aba3559a39f7 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/4aa193391d4a4cbe9d33aba3559a39f7 2024-11-07T09:37:33,913 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/2b58c0f555904d7e930f1857a61a7d75 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/2b58c0f555904d7e930f1857a61a7d75 2024-11-07T09:37:33,914 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/f6903116b51345c8babd0eada216f02d to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/f6903116b51345c8babd0eada216f02d 2024-11-07T09:37:33,915 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/45d6fc8e43944071a62278cd0a1e7ebc to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/45d6fc8e43944071a62278cd0a1e7ebc 2024-11-07T09:37:33,916 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/d75bb0c439da419ebf5558fedbc1e76d to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/d75bb0c439da419ebf5558fedbc1e76d 2024-11-07T09:37:33,916 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/30c9caef9cbe416fb69d974b947c95aa to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/30c9caef9cbe416fb69d974b947c95aa 2024-11-07T09:37:33,917 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/78c512af0fe54c6cb5a3115ab9f8bd1e to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/78c512af0fe54c6cb5a3115ab9f8bd1e 2024-11-07T09:37:33,918 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/157c6d58e9d8403ca2bf402c42f97646 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/157c6d58e9d8403ca2bf402c42f97646 2024-11-07T09:37:33,919 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/ed34b001e7914b6cb45dcb9627954b1b to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/ed34b001e7914b6cb45dcb9627954b1b 2024-11-07T09:37:33,920 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/7ee99ea076304c13accbfced55cf7ab0 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/7ee99ea076304c13accbfced55cf7ab0 2024-11-07T09:37:33,920 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/fdf4034adfa2457e8df7e60dae72f0d2 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/fdf4034adfa2457e8df7e60dae72f0d2 2024-11-07T09:37:33,921 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/468523535622432f9b9a4760d669b7df to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/468523535622432f9b9a4760d669b7df 2024-11-07T09:37:33,922 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/eda6886d0f7c46b58175d585d9c8040a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/eda6886d0f7c46b58175d585d9c8040a 2024-11-07T09:37:33,923 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/d10316279e4b4152944e430b5c2ecc5b to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/d10316279e4b4152944e430b5c2ecc5b 2024-11-07T09:37:33,923 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/c4a909ffe3284dff91e90db6ae06ccad to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/c4a909ffe3284dff91e90db6ae06ccad 2024-11-07T09:37:33,924 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/39a8040f3bfe45dd84059e201d5380e6 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/39a8040f3bfe45dd84059e201d5380e6 2024-11-07T09:37:33,925 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/d0a4c7369e1c430db8ea11dd53193400 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/d0a4c7369e1c430db8ea11dd53193400 2024-11-07T09:37:33,926 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/63471011cc884ce4b60054549d803cdb to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/63471011cc884ce4b60054549d803cdb 2024-11-07T09:37:33,926 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/66951a2517e843998a292a4f6ef4f7df to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/66951a2517e843998a292a4f6ef4f7df 2024-11-07T09:37:33,927 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/3cba0eee27b143a091e8bec2967076da to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/3cba0eee27b143a091e8bec2967076da 2024-11-07T09:37:33,928 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/03c58a40343249e7af5596a53b222a21 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/03c58a40343249e7af5596a53b222a21 2024-11-07T09:37:33,929 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/a3e80b4e1f5e4ab3abc24e84c3f54225 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/a3e80b4e1f5e4ab3abc24e84c3f54225 2024-11-07T09:37:33,930 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/90e1e75c2a7e40fd81c5af551b6d67c5 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/90e1e75c2a7e40fd81c5af551b6d67c5 2024-11-07T09:37:33,931 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/094833d6b09d47cda6cfc4efc49f9ec1 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/094833d6b09d47cda6cfc4efc49f9ec1 2024-11-07T09:37:33,932 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/80d2176ea5f943ca96e7d4255f4b3b95 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/80d2176ea5f943ca96e7d4255f4b3b95 2024-11-07T09:37:33,932 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/1d6bb8ebbcb1440ca49c1687b38099b4 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/1d6bb8ebbcb1440ca49c1687b38099b4 2024-11-07T09:37:33,933 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/d3855aea80e249309d38cba5e145765a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/d3855aea80e249309d38cba5e145765a 2024-11-07T09:37:33,934 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/ad0ef88c8c984215be41618805b16b45 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/ad0ef88c8c984215be41618805b16b45 2024-11-07T09:37:33,935 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/a5aaf00701d94f73ab1d1db1783e2312 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/a5aaf00701d94f73ab1d1db1783e2312 2024-11-07T09:37:33,935 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/e284817689704ca7bf87f56e40c2a074 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/e284817689704ca7bf87f56e40c2a074 2024-11-07T09:37:33,936 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/e17f20c1f1dc4a25bbc474b9686a5c1f to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/e17f20c1f1dc4a25bbc474b9686a5c1f 2024-11-07T09:37:33,937 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/e28d60f8f55c45dabcb8b7174ebe057b to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/e28d60f8f55c45dabcb8b7174ebe057b 2024-11-07T09:37:33,938 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/4f7c1643bb5b4f8e9ddda5cd84f62ae9 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/4f7c1643bb5b4f8e9ddda5cd84f62ae9 2024-11-07T09:37:33,939 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/e7dd334c3de84252a30cbd65d129893e to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/e7dd334c3de84252a30cbd65d129893e 2024-11-07T09:37:33,939 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/39a850f29a224a7fb2badcaaf5cff9b9 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/39a850f29a224a7fb2badcaaf5cff9b9 2024-11-07T09:37:33,941 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/62366126444b4854a8f65a180a02a184, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/a4d5a2ecbc234470b4556be2cdf0b766, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/f5a7a611fe294bc49cba015cfc0c3ed3, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/18cb8f3e32b148708ec3315344bf53a4, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/b72c10544cb04b8b9a4263601a34a73a, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/d1350934596941f7bf5a8aeb25184c17, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/18d5dc17b3d44bc8940944c6f721f0af, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/f8351f23305f4d8ab65ecefe616ff687, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/ac7998e7d8b144dda5f3b41b4d664751, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/5e7709e8e0c34a11a55b40643682f34a, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/36b10cd6f8bf44e590959ae841183aec, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/a21cf5509f6440f285d2462a6c2b554c, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/9c93a880b0464030823dd86c8215b165, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/120008ccfe594daea80078a90b4d6e1a, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/3e19d5ffd7cf4054b608ad07dcad7554, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/75c1345ceb2643f6a979f6c83cc33f8f, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/eb628937099e4163900f73e085f73d2c, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/40d337b9c7584b11ba168e64a1c589f4, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/702396d74fa542328cc882420c761d92, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/390844df72af4820bb5d935815e0f4c2, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/1681f95fbb5141769501952149b07a48, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/1a9e4432cc524e5daaadf24274ce3e11, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/a0ab9018fa414d0aace2ce4bb9d8da38, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/f9b223ebc2fd4c6c8f54ddfcc40cc01e, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/c64b40607a12478fa446ce8a13076a03, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/0d705be7a0d942e196f8cd0fe2f2ac6f, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/652283d14dba47c5ab4dfe832a3d8a03, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/845c5569af8e4db8936c53b9382b0ce1, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/91d8a240b2a944f7b37ed0bb54601cde, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/4c82f22898974ba499980b7600776486, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/0859d533f25440a99eb09717f8e92ea3, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/780be5a3fafd4e59b64ddcd4a13f4578, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/afce29e21c484b56a0fd0fd8b0d5120e, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/0bcebb9ede614f1b8962b4c30cd14fb8, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/83384e9097ee4352817ad2a4654839ec, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/f314ccb21e1942db9fa039fb7f60a4f8, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/4568c06e5c134c3ba14e26b719e7fe66] to archive 2024-11-07T09:37:33,942 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T09:37:33,943 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/62366126444b4854a8f65a180a02a184 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/62366126444b4854a8f65a180a02a184 2024-11-07T09:37:33,944 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/a4d5a2ecbc234470b4556be2cdf0b766 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/a4d5a2ecbc234470b4556be2cdf0b766 2024-11-07T09:37:33,945 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/f5a7a611fe294bc49cba015cfc0c3ed3 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/f5a7a611fe294bc49cba015cfc0c3ed3 2024-11-07T09:37:33,945 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/18cb8f3e32b148708ec3315344bf53a4 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/18cb8f3e32b148708ec3315344bf53a4 2024-11-07T09:37:33,946 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/b72c10544cb04b8b9a4263601a34a73a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/b72c10544cb04b8b9a4263601a34a73a 2024-11-07T09:37:33,947 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/d1350934596941f7bf5a8aeb25184c17 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/d1350934596941f7bf5a8aeb25184c17 2024-11-07T09:37:33,947 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/18d5dc17b3d44bc8940944c6f721f0af to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/18d5dc17b3d44bc8940944c6f721f0af 2024-11-07T09:37:33,948 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/f8351f23305f4d8ab65ecefe616ff687 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/f8351f23305f4d8ab65ecefe616ff687 2024-11-07T09:37:33,949 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/ac7998e7d8b144dda5f3b41b4d664751 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/ac7998e7d8b144dda5f3b41b4d664751 2024-11-07T09:37:33,950 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/5e7709e8e0c34a11a55b40643682f34a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/5e7709e8e0c34a11a55b40643682f34a 2024-11-07T09:37:33,951 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/36b10cd6f8bf44e590959ae841183aec to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/36b10cd6f8bf44e590959ae841183aec 2024-11-07T09:37:33,951 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/a21cf5509f6440f285d2462a6c2b554c to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/a21cf5509f6440f285d2462a6c2b554c 2024-11-07T09:37:33,952 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/9c93a880b0464030823dd86c8215b165 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/9c93a880b0464030823dd86c8215b165 2024-11-07T09:37:33,953 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/120008ccfe594daea80078a90b4d6e1a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/120008ccfe594daea80078a90b4d6e1a 2024-11-07T09:37:33,954 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/3e19d5ffd7cf4054b608ad07dcad7554 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/3e19d5ffd7cf4054b608ad07dcad7554 2024-11-07T09:37:33,954 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/75c1345ceb2643f6a979f6c83cc33f8f to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/75c1345ceb2643f6a979f6c83cc33f8f 2024-11-07T09:37:33,955 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/eb628937099e4163900f73e085f73d2c to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/eb628937099e4163900f73e085f73d2c 2024-11-07T09:37:33,956 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/40d337b9c7584b11ba168e64a1c589f4 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/40d337b9c7584b11ba168e64a1c589f4 2024-11-07T09:37:33,957 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/702396d74fa542328cc882420c761d92 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/702396d74fa542328cc882420c761d92 2024-11-07T09:37:33,958 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/390844df72af4820bb5d935815e0f4c2 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/390844df72af4820bb5d935815e0f4c2 2024-11-07T09:37:33,959 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/1681f95fbb5141769501952149b07a48 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/1681f95fbb5141769501952149b07a48 2024-11-07T09:37:33,959 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/1a9e4432cc524e5daaadf24274ce3e11 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/1a9e4432cc524e5daaadf24274ce3e11 2024-11-07T09:37:33,960 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/a0ab9018fa414d0aace2ce4bb9d8da38 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/a0ab9018fa414d0aace2ce4bb9d8da38 2024-11-07T09:37:33,961 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/f9b223ebc2fd4c6c8f54ddfcc40cc01e to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/f9b223ebc2fd4c6c8f54ddfcc40cc01e 2024-11-07T09:37:33,962 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/c64b40607a12478fa446ce8a13076a03 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/c64b40607a12478fa446ce8a13076a03 2024-11-07T09:37:33,963 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/0d705be7a0d942e196f8cd0fe2f2ac6f to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/0d705be7a0d942e196f8cd0fe2f2ac6f 2024-11-07T09:37:33,964 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/652283d14dba47c5ab4dfe832a3d8a03 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/652283d14dba47c5ab4dfe832a3d8a03 2024-11-07T09:37:33,964 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/845c5569af8e4db8936c53b9382b0ce1 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/845c5569af8e4db8936c53b9382b0ce1 2024-11-07T09:37:33,965 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/91d8a240b2a944f7b37ed0bb54601cde to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/91d8a240b2a944f7b37ed0bb54601cde 2024-11-07T09:37:33,966 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/4c82f22898974ba499980b7600776486 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/4c82f22898974ba499980b7600776486 2024-11-07T09:37:33,967 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/0859d533f25440a99eb09717f8e92ea3 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/0859d533f25440a99eb09717f8e92ea3 2024-11-07T09:37:33,968 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/780be5a3fafd4e59b64ddcd4a13f4578 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/780be5a3fafd4e59b64ddcd4a13f4578 2024-11-07T09:37:33,968 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/afce29e21c484b56a0fd0fd8b0d5120e to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/afce29e21c484b56a0fd0fd8b0d5120e 2024-11-07T09:37:33,969 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/0bcebb9ede614f1b8962b4c30cd14fb8 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/0bcebb9ede614f1b8962b4c30cd14fb8 2024-11-07T09:37:33,970 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/83384e9097ee4352817ad2a4654839ec to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/83384e9097ee4352817ad2a4654839ec 2024-11-07T09:37:33,971 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/f314ccb21e1942db9fa039fb7f60a4f8 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/f314ccb21e1942db9fa039fb7f60a4f8 2024-11-07T09:37:33,971 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/4568c06e5c134c3ba14e26b719e7fe66 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/4568c06e5c134c3ba14e26b719e7fe66 2024-11-07T09:37:33,973 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/39df6e17262b4f5eae374674548e744d, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/ad0dd1c0f59641f484f4351e0c2cba40, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/6e9bc4bd75bd4ebfbb6e9f8e058d39c8, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/4884e1731666416aa699d4f36d08e8de, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/bb1982c8ac434b918cfb128749d9c913, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/e3d86abb5edd4bf98bfb2cfcc8609a8d, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/175bdae664884b6baa57df81c0bbd195, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/3edf0a5a2c074ba59311f7c41f312b37, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/76a80e9d95bc48ab903417fd5fba44c8, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/553e92525a014504b227e284d6b795b9, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/bfc4e336017c4ce7a18a1ebf73784544, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/5f5f0c839ef34f80bb281b6fd80bec40, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/aab565688d664ead8cfdb44eb9d6a626, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/1f9d5d0470c049d69b09a9391e4bda19, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/152ec1b809ec4cc3b79f6f1a47926229, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/841b9026c0e145acaa92fecb493e0093, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/0da2ebedcd904e11a933d9413d9b13c1, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/f1c28f9726d641a7bd453d2d3ccdfe7a, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/2464a659813545c3bf045e05cf07c653, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/3b7019ff7dde4051a6ed0336eafdce10, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/486a4ce51546400c96bbf2a52eb59e8e, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/36321d6abbb9438aa47660e166c92be0, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/8707b54ccd234dc7b81e0b0e47da7f50, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/d97fecfee8134c539dfa942761827187, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/a780b9b7ca0648fc821759ab3ced8fc1, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/afdb7b58ec924e8caa3f813861afe2ad, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/4004364b515b47e181b77c7f16fdd7ef, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/e85f51f60e7f468b8ff0449c2594e95f, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/d7c657e5e0aa4f5a9f8518ad878cb8ae, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/0f534407b4a4428c9a7a87ccca738aa5, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/77825b31686b4c66a93cfe14cca4349d, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/70e41c07fb6d42f983d80c662d65aa2d, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/8a453281d6ea47dea1b1d1144119cd68, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/918b911c25d44771886a01f965bb7dce, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/989a3731eadf480a98d39ffe9a8734c3, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/5b8246936e5c455bab712c30000491fe] to archive 2024-11-07T09:37:33,974 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T09:37:33,975 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/39df6e17262b4f5eae374674548e744d to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/39df6e17262b4f5eae374674548e744d 2024-11-07T09:37:33,976 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/ad0dd1c0f59641f484f4351e0c2cba40 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/ad0dd1c0f59641f484f4351e0c2cba40 2024-11-07T09:37:33,976 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/6e9bc4bd75bd4ebfbb6e9f8e058d39c8 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/6e9bc4bd75bd4ebfbb6e9f8e058d39c8 2024-11-07T09:37:33,977 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/4884e1731666416aa699d4f36d08e8de to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/4884e1731666416aa699d4f36d08e8de 2024-11-07T09:37:33,978 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/bb1982c8ac434b918cfb128749d9c913 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/bb1982c8ac434b918cfb128749d9c913 2024-11-07T09:37:33,978 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/e3d86abb5edd4bf98bfb2cfcc8609a8d to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/e3d86abb5edd4bf98bfb2cfcc8609a8d 2024-11-07T09:37:33,979 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/175bdae664884b6baa57df81c0bbd195 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/175bdae664884b6baa57df81c0bbd195 2024-11-07T09:37:33,980 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/3edf0a5a2c074ba59311f7c41f312b37 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/3edf0a5a2c074ba59311f7c41f312b37 2024-11-07T09:37:33,981 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/76a80e9d95bc48ab903417fd5fba44c8 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/76a80e9d95bc48ab903417fd5fba44c8 2024-11-07T09:37:33,981 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/553e92525a014504b227e284d6b795b9 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/553e92525a014504b227e284d6b795b9 2024-11-07T09:37:33,982 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/bfc4e336017c4ce7a18a1ebf73784544 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/bfc4e336017c4ce7a18a1ebf73784544 2024-11-07T09:37:33,983 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/5f5f0c839ef34f80bb281b6fd80bec40 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/5f5f0c839ef34f80bb281b6fd80bec40 2024-11-07T09:37:33,983 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/aab565688d664ead8cfdb44eb9d6a626 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/aab565688d664ead8cfdb44eb9d6a626 2024-11-07T09:37:33,984 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/1f9d5d0470c049d69b09a9391e4bda19 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/1f9d5d0470c049d69b09a9391e4bda19 2024-11-07T09:37:33,985 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/152ec1b809ec4cc3b79f6f1a47926229 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/152ec1b809ec4cc3b79f6f1a47926229 2024-11-07T09:37:33,986 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/841b9026c0e145acaa92fecb493e0093 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/841b9026c0e145acaa92fecb493e0093 2024-11-07T09:37:33,987 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/0da2ebedcd904e11a933d9413d9b13c1 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/0da2ebedcd904e11a933d9413d9b13c1 2024-11-07T09:37:33,987 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/f1c28f9726d641a7bd453d2d3ccdfe7a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/f1c28f9726d641a7bd453d2d3ccdfe7a 2024-11-07T09:37:33,988 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/2464a659813545c3bf045e05cf07c653 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/2464a659813545c3bf045e05cf07c653 2024-11-07T09:37:33,989 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/3b7019ff7dde4051a6ed0336eafdce10 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/3b7019ff7dde4051a6ed0336eafdce10 2024-11-07T09:37:33,990 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/486a4ce51546400c96bbf2a52eb59e8e to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/486a4ce51546400c96bbf2a52eb59e8e 2024-11-07T09:37:33,990 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/36321d6abbb9438aa47660e166c92be0 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/36321d6abbb9438aa47660e166c92be0 2024-11-07T09:37:33,991 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/8707b54ccd234dc7b81e0b0e47da7f50 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/8707b54ccd234dc7b81e0b0e47da7f50 2024-11-07T09:37:33,992 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/d97fecfee8134c539dfa942761827187 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/d97fecfee8134c539dfa942761827187 2024-11-07T09:37:33,993 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/a780b9b7ca0648fc821759ab3ced8fc1 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/a780b9b7ca0648fc821759ab3ced8fc1 2024-11-07T09:37:33,994 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/afdb7b58ec924e8caa3f813861afe2ad to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/afdb7b58ec924e8caa3f813861afe2ad 2024-11-07T09:37:33,994 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/4004364b515b47e181b77c7f16fdd7ef to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/4004364b515b47e181b77c7f16fdd7ef 2024-11-07T09:37:33,995 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/e85f51f60e7f468b8ff0449c2594e95f to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/e85f51f60e7f468b8ff0449c2594e95f 2024-11-07T09:37:33,996 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/d7c657e5e0aa4f5a9f8518ad878cb8ae to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/d7c657e5e0aa4f5a9f8518ad878cb8ae 2024-11-07T09:37:33,997 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/0f534407b4a4428c9a7a87ccca738aa5 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/0f534407b4a4428c9a7a87ccca738aa5 2024-11-07T09:37:33,997 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/77825b31686b4c66a93cfe14cca4349d to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/77825b31686b4c66a93cfe14cca4349d 2024-11-07T09:37:33,998 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/70e41c07fb6d42f983d80c662d65aa2d to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/70e41c07fb6d42f983d80c662d65aa2d 2024-11-07T09:37:33,999 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/8a453281d6ea47dea1b1d1144119cd68 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/8a453281d6ea47dea1b1d1144119cd68 2024-11-07T09:37:34,000 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/918b911c25d44771886a01f965bb7dce to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/918b911c25d44771886a01f965bb7dce 2024-11-07T09:37:34,001 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/989a3731eadf480a98d39ffe9a8734c3 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/989a3731eadf480a98d39ffe9a8734c3 2024-11-07T09:37:34,002 DEBUG [StoreCloser-TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/5b8246936e5c455bab712c30000491fe to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/5b8246936e5c455bab712c30000491fe 2024-11-07T09:37:34,005 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/recovered.edits/549.seqid, newMaxSeqId=549, maxSeqId=1 2024-11-07T09:37:34,006 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. 2024-11-07T09:37:34,006 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1676): Region close journal for 5165da7af32dbdfd119fd1fa7507f91f: Waiting for close lock at 1730972252666Running coprocessor pre-close hooks at 1730972252666Disabling compacts and flushes for region at 1730972252666Disabling writes for close at 1730972252666Obtaining lock to block concurrent updates at 1730972252667 (+1 ms)Preparing flush snapshotting stores in 5165da7af32dbdfd119fd1fa7507f91f at 1730972252667Finished memstore snapshotting TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1730972252667Flushing stores of TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f. at 1730972252667Flushing 5165da7af32dbdfd119fd1fa7507f91f/A: creating writer at 1730972252667Flushing 5165da7af32dbdfd119fd1fa7507f91f/A: appending metadata at 1730972252670 (+3 ms)Flushing 5165da7af32dbdfd119fd1fa7507f91f/A: closing flushed file at 1730972252670Flushing 5165da7af32dbdfd119fd1fa7507f91f/B: creating writer at 1730972253078 (+408 ms)Flushing 5165da7af32dbdfd119fd1fa7507f91f/B: appending metadata at 1730972253080 (+2 ms)Flushing 5165da7af32dbdfd119fd1fa7507f91f/B: closing flushed file at 1730972253080Flushing 5165da7af32dbdfd119fd1fa7507f91f/C: creating writer at 1730972253488 (+408 ms)Flushing 5165da7af32dbdfd119fd1fa7507f91f/C: appending metadata at 1730972253490 (+2 ms)Flushing 5165da7af32dbdfd119fd1fa7507f91f/C: closing flushed file at 1730972253490Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@55de526d: reopening flushed file at 1730972253897 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c399646: reopening flushed file at 1730972253900 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7acf9cad: reopening flushed file at 1730972253903 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 5165da7af32dbdfd119fd1fa7507f91f in 1241ms, sequenceid=546, compaction requested=true at 1730972253907 (+4 ms)Writing region close event to WAL at 1730972254003 (+96 ms)Running coprocessor post-close hooks at 1730972254006 (+3 ms)Closed at 1730972254006 2024-11-07T09:37:34,007 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(157): Closed 5165da7af32dbdfd119fd1fa7507f91f 2024-11-07T09:37:34,008 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=5165da7af32dbdfd119fd1fa7507f91f, regionState=CLOSED 2024-11-07T09:37:34,009 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 because future has completed 2024-11-07T09:37:34,011 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-11-07T09:37:34,011 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; CloseRegionProcedure 5165da7af32dbdfd119fd1fa7507f91f, server=9f1eb259a29b,42895,1730972153970 in 1.4960 sec 2024-11-07T09:37:34,012 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-07T09:37:34,012 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5165da7af32dbdfd119fd1fa7507f91f, UNASSIGN in 1.5000 sec 2024-11-07T09:37:34,014 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-07T09:37:34,014 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5030 sec 2024-11-07T09:37:34,015 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730972254015"}]},"ts":"1730972254015"} 2024-11-07T09:37:34,016 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-07T09:37:34,016 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-07T09:37:34,017 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5100 sec 2024-11-07T09:37:34,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-07T09:37:34,653 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:37:34,653 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-07T09:37:34,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=98, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T09:37:34,655 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=98, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T09:37:34,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=98 2024-11-07T09:37:34,655 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=98, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T09:37:34,658 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f 2024-11-07T09:37:34,660 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A, FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B, FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C, FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/recovered.edits] 2024-11-07T09:37:34,662 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/490aeacd52cd4ba883acf2b949f0d0e5 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/490aeacd52cd4ba883acf2b949f0d0e5 2024-11-07T09:37:34,663 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/602491f7d1ed41b0a7b5d7c0be59d770 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/602491f7d1ed41b0a7b5d7c0be59d770 2024-11-07T09:37:34,664 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/949c163054c04419807fc49fff84677d to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/A/949c163054c04419807fc49fff84677d 2024-11-07T09:37:34,666 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/11c6b92ab3e2469cb0fbc2b6da8a782c to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/11c6b92ab3e2469cb0fbc2b6da8a782c 2024-11-07T09:37:34,667 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/909333a2aea641d585f2570e225e4346 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/909333a2aea641d585f2570e225e4346 2024-11-07T09:37:34,667 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/b6d61e4dcf9a42e29f44fff0162062e4 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/B/b6d61e4dcf9a42e29f44fff0162062e4 2024-11-07T09:37:34,669 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/62a53f48b86249d8b590157cd5028eb1 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/62a53f48b86249d8b590157cd5028eb1 2024-11-07T09:37:34,670 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/6cb2809ffdd949f49513edb91de16974 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/6cb2809ffdd949f49513edb91de16974 2024-11-07T09:37:34,671 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/8abd6e0fe1fc497e845704eebf688776 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/C/8abd6e0fe1fc497e845704eebf688776 2024-11-07T09:37:34,673 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/recovered.edits/549.seqid to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f/recovered.edits/549.seqid 2024-11-07T09:37:34,674 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/5165da7af32dbdfd119fd1fa7507f91f 2024-11-07T09:37:34,674 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-07T09:37:34,675 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=98, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T09:37:34,678 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-07T09:37:34,680 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-07T09:37:34,681 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=98, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T09:37:34,681 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-07T09:37:34,681 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1730972254681"}]},"ts":"9223372036854775807"} 2024-11-07T09:37:34,683 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-07T09:37:34,683 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 5165da7af32dbdfd119fd1fa7507f91f, NAME => 'TestAcidGuarantees,,1730972221061.5165da7af32dbdfd119fd1fa7507f91f.', STARTKEY => '', ENDKEY => ''}] 2024-11-07T09:37:34,683 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-07T09:37:34,683 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1730972254683"}]},"ts":"9223372036854775807"} 2024-11-07T09:37:34,684 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-07T09:37:34,685 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=98, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T09:37:34,686 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 32 msec 2024-11-07T09:37:34,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=98 2024-11-07T09:37:34,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-07T09:37:34,764 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:37:34,774 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=237 (was 238), OpenFileDescriptor=447 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=355 (was 385), ProcessCount=11 (was 11), AvailableMemoryMB=4466 (was 4725) 2024-11-07T09:37:34,783 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=237, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=355, ProcessCount=11, AvailableMemoryMB=4466 2024-11-07T09:37:34,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T09:37:34,784 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T09:37:34,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-07T09:37:34,786 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=99, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T09:37:34,786 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:34,786 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 99 2024-11-07T09:37:34,787 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=99, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T09:37:34,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-07T09:37:34,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742200_1376 (size=959) 2024-11-07T09:37:34,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-07T09:37:35,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-07T09:37:35,194 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260 2024-11-07T09:37:35,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742201_1377 (size=53) 2024-11-07T09:37:35,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-07T09:37:35,599 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T09:37:35,599 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 7f7127bc07c9cd4d3f409040fbdd89d3, disabling compactions & flushes 2024-11-07T09:37:35,599 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:35,599 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:35,599 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. after waiting 0 ms 2024-11-07T09:37:35,599 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:35,599 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:35,599 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7f7127bc07c9cd4d3f409040fbdd89d3: Waiting for close lock at 1730972255599Disabling compacts and flushes for region at 1730972255599Disabling writes for close at 1730972255599Writing region close event to WAL at 1730972255599Closed at 1730972255599 2024-11-07T09:37:35,600 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=99, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T09:37:35,600 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1730972255600"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730972255600"}]},"ts":"1730972255600"} 2024-11-07T09:37:35,602 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-07T09:37:35,602 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=99, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T09:37:35,602 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730972255602"}]},"ts":"1730972255602"} 2024-11-07T09:37:35,603 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-07T09:37:35,604 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7f7127bc07c9cd4d3f409040fbdd89d3, ASSIGN}] 2024-11-07T09:37:35,604 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7f7127bc07c9cd4d3f409040fbdd89d3, ASSIGN 2024-11-07T09:37:35,605 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7f7127bc07c9cd4d3f409040fbdd89d3, ASSIGN; state=OFFLINE, location=9f1eb259a29b,42895,1730972153970; forceNewPlan=false, retain=false 2024-11-07T09:37:35,755 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=100 updating hbase:meta row=7f7127bc07c9cd4d3f409040fbdd89d3, regionState=OPENING, regionLocation=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:35,757 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7f7127bc07c9cd4d3f409040fbdd89d3, ASSIGN because future has completed 2024-11-07T09:37:35,757 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=101, ppid=100, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970}] 2024-11-07T09:37:35,911 INFO [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:35,912 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(7752): Opening region: {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} 2024-11-07T09:37:35,912 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:35,912 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T09:37:35,912 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(7794): checking encryption for 7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:35,912 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(7797): checking classloading for 7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:35,913 INFO [StoreOpener-7f7127bc07c9cd4d3f409040fbdd89d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:35,914 INFO [StoreOpener-7f7127bc07c9cd4d3f409040fbdd89d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7f7127bc07c9cd4d3f409040fbdd89d3 columnFamilyName A 2024-11-07T09:37:35,914 DEBUG [StoreOpener-7f7127bc07c9cd4d3f409040fbdd89d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:35,915 INFO [StoreOpener-7f7127bc07c9cd4d3f409040fbdd89d3-1 {}] regionserver.HStore(327): Store=7f7127bc07c9cd4d3f409040fbdd89d3/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T09:37:35,915 INFO [StoreOpener-7f7127bc07c9cd4d3f409040fbdd89d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:35,916 INFO [StoreOpener-7f7127bc07c9cd4d3f409040fbdd89d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7f7127bc07c9cd4d3f409040fbdd89d3 columnFamilyName B 2024-11-07T09:37:35,916 DEBUG [StoreOpener-7f7127bc07c9cd4d3f409040fbdd89d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:35,916 INFO [StoreOpener-7f7127bc07c9cd4d3f409040fbdd89d3-1 {}] regionserver.HStore(327): Store=7f7127bc07c9cd4d3f409040fbdd89d3/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T09:37:35,916 INFO [StoreOpener-7f7127bc07c9cd4d3f409040fbdd89d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:35,917 INFO [StoreOpener-7f7127bc07c9cd4d3f409040fbdd89d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7f7127bc07c9cd4d3f409040fbdd89d3 columnFamilyName C 2024-11-07T09:37:35,917 DEBUG [StoreOpener-7f7127bc07c9cd4d3f409040fbdd89d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:35,917 INFO [StoreOpener-7f7127bc07c9cd4d3f409040fbdd89d3-1 {}] regionserver.HStore(327): Store=7f7127bc07c9cd4d3f409040fbdd89d3/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T09:37:35,917 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(1038): replaying wal for 7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:35,918 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:35,918 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:35,919 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(1048): stopping wal replay for 7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:35,919 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(1060): Cleaning up temporary data for 7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:35,919 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T09:37:35,920 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(1093): writing seq id for 7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:35,921 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T09:37:35,922 INFO [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(1114): Opened 7f7127bc07c9cd4d3f409040fbdd89d3; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72300068, jitterRate=0.07735496759414673}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T09:37:35,922 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:35,922 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(1006): Region open journal for 7f7127bc07c9cd4d3f409040fbdd89d3: Running coprocessor pre-open hook at 1730972255912Writing region info on filesystem at 1730972255912Initializing all the Stores at 1730972255913 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730972255913Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730972255913Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730972255913Cleaning up temporary data from old regions at 1730972255919 (+6 ms)Running coprocessor post-open hooks at 1730972255922 (+3 ms)Region opened successfully at 1730972255922 2024-11-07T09:37:35,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-07T09:37:35,923 INFO [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., pid=101, masterSystemTime=1730972255909 2024-11-07T09:37:35,925 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:35,925 INFO [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:35,925 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=100 updating hbase:meta row=7f7127bc07c9cd4d3f409040fbdd89d3, regionState=OPEN, openSeqNum=2, regionLocation=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:35,926 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=101, ppid=100, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 because future has completed 2024-11-07T09:37:35,929 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=101, resume processing ppid=100 2024-11-07T09:37:35,929 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, ppid=100, state=SUCCESS, hasLock=false; OpenRegionProcedure 7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 in 170 msec 2024-11-07T09:37:35,930 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-07T09:37:35,930 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7f7127bc07c9cd4d3f409040fbdd89d3, ASSIGN in 325 msec 2024-11-07T09:37:35,931 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=99, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T09:37:35,931 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730972255931"}]},"ts":"1730972255931"} 2024-11-07T09:37:35,932 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-07T09:37:35,933 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=99, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T09:37:35,934 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1490 sec 2024-11-07T09:37:36,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-07T09:37:36,933 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:37:36,934 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ab2a0ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:36,934 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:37:36,934 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:37:36,935 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:37:36,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:37:36,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:37:36,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@728d9aed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:36,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:37:36,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:37:36,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:36,937 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48110, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:37:36,938 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@597b212e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:36,938 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:37:36,938 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:37:36,939 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:37:36,940 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49076, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:37:36,940 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T09:37:36,941 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 9f1eb259a29b,41689,1730972153310 2024-11-07T09:37:36,941 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@77fc21b8 2024-11-07T09:37:36,941 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T09:37:36,942 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48120, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T09:37:36,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T09:37:36,943 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T09:37:36,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=102, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-07T09:37:36,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742202_1378 (size=995) 2024-11-07T09:37:37,354 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-07T09:37:37,354 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-07T09:37:37,356 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=103, ppid=102, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T09:37:37,358 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7f7127bc07c9cd4d3f409040fbdd89d3, REOPEN/MOVE}] 2024-11-07T09:37:37,359 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=104, ppid=103, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7f7127bc07c9cd4d3f409040fbdd89d3, REOPEN/MOVE 2024-11-07T09:37:37,359 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=104 updating hbase:meta row=7f7127bc07c9cd4d3f409040fbdd89d3, regionState=CLOSING, regionLocation=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:37,361 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=104, ppid=103, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7f7127bc07c9cd4d3f409040fbdd89d3, REOPEN/MOVE because future has completed 2024-11-07T09:37:37,361 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T09:37:37,361 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=105, ppid=104, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970}] 2024-11-07T09:37:37,513 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] handler.UnassignRegionHandler(122): Close 7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:37,513 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T09:37:37,513 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegion(1722): Closing 7f7127bc07c9cd4d3f409040fbdd89d3, disabling compactions & flushes 2024-11-07T09:37:37,513 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:37,513 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:37,513 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. after waiting 0 ms 2024-11-07T09:37:37,513 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:37,517 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T09:37:37,517 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:37,517 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegion(1676): Region close journal for 7f7127bc07c9cd4d3f409040fbdd89d3: Waiting for close lock at 1730972257513Running coprocessor pre-close hooks at 1730972257513Disabling compacts and flushes for region at 1730972257513Disabling writes for close at 1730972257513Writing region close event to WAL at 1730972257514 (+1 ms)Running coprocessor post-close hooks at 1730972257517 (+3 ms)Closed at 1730972257517 2024-11-07T09:37:37,517 WARN [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegionServer(3299): Not adding moved region record: 7f7127bc07c9cd4d3f409040fbdd89d3 to self. 2024-11-07T09:37:37,519 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] handler.UnassignRegionHandler(157): Closed 7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:37,519 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=104 updating hbase:meta row=7f7127bc07c9cd4d3f409040fbdd89d3, regionState=CLOSED 2024-11-07T09:37:37,521 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=105, ppid=104, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 because future has completed 2024-11-07T09:37:37,522 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=105, resume processing ppid=104 2024-11-07T09:37:37,522 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, ppid=104, state=SUCCESS, hasLock=false; CloseRegionProcedure 7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 in 160 msec 2024-11-07T09:37:37,523 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=104, ppid=103, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7f7127bc07c9cd4d3f409040fbdd89d3, REOPEN/MOVE; state=CLOSED, location=9f1eb259a29b,42895,1730972153970; forceNewPlan=false, retain=true 2024-11-07T09:37:37,674 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=104 updating hbase:meta row=7f7127bc07c9cd4d3f409040fbdd89d3, regionState=OPENING, regionLocation=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:37,675 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=104, ppid=103, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7f7127bc07c9cd4d3f409040fbdd89d3, REOPEN/MOVE because future has completed 2024-11-07T09:37:37,675 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=104, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970}] 2024-11-07T09:37:37,829 INFO [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:37,829 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7752): Opening region: {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} 2024-11-07T09:37:37,830 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:37,830 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T09:37:37,830 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7794): checking encryption for 7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:37,830 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7797): checking classloading for 7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:37,831 INFO [StoreOpener-7f7127bc07c9cd4d3f409040fbdd89d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:37,832 INFO [StoreOpener-7f7127bc07c9cd4d3f409040fbdd89d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7f7127bc07c9cd4d3f409040fbdd89d3 columnFamilyName A 2024-11-07T09:37:37,833 DEBUG [StoreOpener-7f7127bc07c9cd4d3f409040fbdd89d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:37,833 INFO [StoreOpener-7f7127bc07c9cd4d3f409040fbdd89d3-1 {}] regionserver.HStore(327): Store=7f7127bc07c9cd4d3f409040fbdd89d3/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T09:37:37,833 INFO [StoreOpener-7f7127bc07c9cd4d3f409040fbdd89d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:37,834 INFO [StoreOpener-7f7127bc07c9cd4d3f409040fbdd89d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7f7127bc07c9cd4d3f409040fbdd89d3 columnFamilyName B 2024-11-07T09:37:37,834 DEBUG [StoreOpener-7f7127bc07c9cd4d3f409040fbdd89d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:37,834 INFO [StoreOpener-7f7127bc07c9cd4d3f409040fbdd89d3-1 {}] regionserver.HStore(327): Store=7f7127bc07c9cd4d3f409040fbdd89d3/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T09:37:37,834 INFO [StoreOpener-7f7127bc07c9cd4d3f409040fbdd89d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:37,835 INFO [StoreOpener-7f7127bc07c9cd4d3f409040fbdd89d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7f7127bc07c9cd4d3f409040fbdd89d3 columnFamilyName C 2024-11-07T09:37:37,835 DEBUG [StoreOpener-7f7127bc07c9cd4d3f409040fbdd89d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:37,835 INFO [StoreOpener-7f7127bc07c9cd4d3f409040fbdd89d3-1 {}] regionserver.HStore(327): Store=7f7127bc07c9cd4d3f409040fbdd89d3/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T09:37:37,835 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1038): replaying wal for 7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:37,836 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:37,837 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:37,837 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1048): stopping wal replay for 7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:37,837 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1060): Cleaning up temporary data for 7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:37,838 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T09:37:37,839 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1093): writing seq id for 7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:37,839 INFO [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1114): Opened 7f7127bc07c9cd4d3f409040fbdd89d3; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67226472, jitterRate=0.0017524957656860352}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T09:37:37,839 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:37,840 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1006): Region open journal for 7f7127bc07c9cd4d3f409040fbdd89d3: Running coprocessor pre-open hook at 1730972257830Writing region info on filesystem at 1730972257830Initializing all the Stores at 1730972257831 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730972257831Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730972257831Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730972257831Cleaning up temporary data from old regions at 1730972257837 (+6 ms)Running coprocessor post-open hooks at 1730972257839 (+2 ms)Region opened successfully at 1730972257840 (+1 ms) 2024-11-07T09:37:37,841 INFO [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., pid=106, masterSystemTime=1730972257827 2024-11-07T09:37:37,842 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:37,842 INFO [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:37,843 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=104 updating hbase:meta row=7f7127bc07c9cd4d3f409040fbdd89d3, regionState=OPEN, openSeqNum=5, regionLocation=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:37,844 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=106, ppid=104, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 because future has completed 2024-11-07T09:37:37,846 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=104 2024-11-07T09:37:37,846 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=104, state=SUCCESS, hasLock=false; OpenRegionProcedure 7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 in 170 msec 2024-11-07T09:37:37,847 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-07T09:37:37,847 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7f7127bc07c9cd4d3f409040fbdd89d3, REOPEN/MOVE in 488 msec 2024-11-07T09:37:37,849 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=103, resume processing ppid=102 2024-11-07T09:37:37,849 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, ppid=102, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 492 msec 2024-11-07T09:37:37,851 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 907 msec 2024-11-07T09:37:37,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=102 2024-11-07T09:37:37,852 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:37:37,852 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38802cb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:37,852 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:37:37,853 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:37:37,853 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:37:37,853 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:37:37,854 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:37:37,854 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2456814f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:37,854 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:37:37,854 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:37:37,854 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:37,855 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48136, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:37:37,855 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d204ce7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:37,856 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@209ba84b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:37,856 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:37:37,856 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:37:37,856 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:37:37,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:37:37,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:37:37,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26230b8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:37,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:37:37,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:37:37,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:37,858 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48154, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:37:37,858 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f6e4e5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:37,858 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7495e76b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:37,859 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:37:37,859 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:37:37,859 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:37:37,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:37:37,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:37:37,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ddef26, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:37,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:37:37,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:37:37,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:37,860 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48162, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:37:37,861 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a18d7be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:37,861 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7288b4d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:37,861 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:37:37,861 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:37:37,862 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:37:37,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:37:37,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:37:37,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3589f3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:37,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:37:37,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:37:37,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:37,863 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48180, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:37:37,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6965bb05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:37,864 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69edc398, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:37,864 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:37:37,864 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:37:37,865 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:37:37,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:37:37,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:37:37,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@409cc52c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:37,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:37:37,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:37:37,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:37,866 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48200, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:37:37,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5014408e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:37,867 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62931224, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:37,867 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:37:37,867 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:37:37,867 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:37:37,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:37:37,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:37:37,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b5c1f02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:37,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:37:37,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:37:37,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:37,868 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48214, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:37:37,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@407dc66d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:37,869 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@400fc310, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:37,869 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:37:37,869 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:37:37,870 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:37:37,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:37:37,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:37:37,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a640833, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:37,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:37:37,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:37:37,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:37,871 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48226, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:37:37,871 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58351cd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:37,872 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@227d6625, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:37,872 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:37:37,872 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:37:37,872 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:37:37,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:37:37,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:37:37,873 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66f8f745, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:37,873 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:37:37,873 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:37:37,873 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:37,873 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48250, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:37:37,874 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e5c4162, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:37,874 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15113514, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:37,874 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:37:37,874 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:37:37,875 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:37:37,875 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:37:37,875 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:37:37,875 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4074db81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:37,875 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:37:37,875 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:37:37,875 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:37,876 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48268, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:37:37,876 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7da97f01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:37,877 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ed8e2d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:37,877 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:37:37,877 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:37:37,878 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:37:37,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:37:37,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:37:37,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cff1a4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:37,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:37:37,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:37:37,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:37,879 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48288, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:37:37,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dcf908d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:37:37,880 DEBUG [Thread-1722 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:37:37,880 DEBUG [Thread-1724 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:37:37,880 DEBUG [Thread-1705 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:37:37,880 DEBUG [Thread-1711 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:37:37,880 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T09:37:37,880 DEBUG [Thread-1713 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:37:37,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:37:37,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:37:37,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:37:37,881 DEBUG [Thread-1707 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:37:37,881 DEBUG [Thread-1718 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:37:37,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:37:37,881 DEBUG [Thread-1716 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:37:37,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:37:37,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:37:37,881 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:37:37,881 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:37:37,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-07T09:37:37,882 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:37:37,882 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:37:37,882 DEBUG [Thread-1709 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:37:37,882 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49098, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:37:37,882 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49086, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:37:37,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:37:37,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:37:37,883 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49104, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:37:37,883 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49116, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:37:37,883 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49126, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:37:37,883 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:37:37,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-07T09:37:37,883 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:37:37,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:37:37,883 DEBUG [Thread-1720 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:37:37,884 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:37:37,884 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:37:37,884 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:37:37,884 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49140, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:37:37,884 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T09:37:37,885 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49146, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:37:37,885 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49148, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:37:37,885 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49162, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:37:37,885 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T09:37:37,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5] 2024-11-07T09:37:37,885 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T09:37:37,885 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:37:37,886 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:37:37,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5] 2024-11-07T09:37:37,887 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49172, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:37:37,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5] 2024-11-07T09:37:37,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5] 2024-11-07T09:37:37,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5] 2024-11-07T09:37:37,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5] 2024-11-07T09:37:37,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5] 2024-11-07T09:37:37,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5] 2024-11-07T09:37:37,892 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5] 2024-11-07T09:37:37,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5] 2024-11-07T09:37:37,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:37,896 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:37:37,897 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7f7127bc07c9cd4d3f409040fbdd89d3 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T09:37:37,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107e7b1b11c680b4b5a9b3b9fba23742d27_7f7127bc07c9cd4d3f409040fbdd89d3 is 50, key is test_row_0/A:col10/1730972257894/Put/seqid=0 2024-11-07T09:37:37,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742203_1379 (size=17034) 2024-11-07T09:37:37,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:37,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49162 deadline: 1730972317934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:37,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:37,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:37,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:37,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:37,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49116 deadline: 1730972317934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:37,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:37,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:37,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:37,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:37,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49098 deadline: 1730972317935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:37,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:37,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:37,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49148 deadline: 1730972317936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:37,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:37,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:37,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:37,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:37,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:37,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:37,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49086 deadline: 1730972317946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:37,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:37,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:37,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:37,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-07T09:37:38,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-07T09:37:38,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:38,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. as already flushing 2024-11-07T09:37:38,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:38,038 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:38,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:38,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:38,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:38,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49116 deadline: 1730972318053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:38,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:38,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:38,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:38,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:38,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49098 deadline: 1730972318053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:38,057 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:38,057 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:38,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:38,057 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:38,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49162 deadline: 1730972318053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:38,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:38,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:38,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:38,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:38,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49148 deadline: 1730972318053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:38,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:38,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:38,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:38,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:38,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49086 deadline: 1730972318053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:38,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:38,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:38,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:38,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-07T09:37:38,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:38,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. as already flushing 2024-11-07T09:37:38,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:38,191 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:38,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:38,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:38,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-07T09:37:38,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:38,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49116 deadline: 1730972318263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:38,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:38,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:38,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49098 deadline: 1730972318263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:38,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:38,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:38,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:38,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:38,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49162 deadline: 1730972318263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:38,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:38,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:38,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:38,271 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:38,271 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:38,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49148 deadline: 1730972318264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:38,271 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:38,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:38,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:38,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49086 deadline: 1730972318264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:38,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:38,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:38,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:38,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:38,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:38,331 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:38,335 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107e7b1b11c680b4b5a9b3b9fba23742d27_7f7127bc07c9cd4d3f409040fbdd89d3 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107e7b1b11c680b4b5a9b3b9fba23742d27_7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:38,336 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/0d4b997726f14c8396e95dfa0ff94968, store: [table=TestAcidGuarantees family=A region=7f7127bc07c9cd4d3f409040fbdd89d3] 2024-11-07T09:37:38,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/0d4b997726f14c8396e95dfa0ff94968 is 175, key is test_row_0/A:col10/1730972257894/Put/seqid=0 2024-11-07T09:37:38,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742204_1380 (size=48139) 2024-11-07T09:37:38,340 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/0d4b997726f14c8396e95dfa0ff94968 2024-11-07T09:37:38,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-07T09:37:38,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:38,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. as already flushing 2024-11-07T09:37:38,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:38,344 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:38,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:38,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:38,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/effc86c3e9914434b79bc59a0873480d is 50, key is test_row_0/B:col10/1730972257894/Put/seqid=0 2024-11-07T09:37:38,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742205_1381 (size=12001) 2024-11-07T09:37:38,367 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/effc86c3e9914434b79bc59a0873480d 2024-11-07T09:37:38,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/c5d0b0e8a858466a8232572debb86dfd is 50, key is test_row_0/C:col10/1730972257894/Put/seqid=0 2024-11-07T09:37:38,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742206_1382 (size=12001) 2024-11-07T09:37:38,408 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/c5d0b0e8a858466a8232572debb86dfd 2024-11-07T09:37:38,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/0d4b997726f14c8396e95dfa0ff94968 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/0d4b997726f14c8396e95dfa0ff94968 2024-11-07T09:37:38,422 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/0d4b997726f14c8396e95dfa0ff94968, entries=250, sequenceid=16, filesize=47.0 K 2024-11-07T09:37:38,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/effc86c3e9914434b79bc59a0873480d as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/effc86c3e9914434b79bc59a0873480d 2024-11-07T09:37:38,426 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/effc86c3e9914434b79bc59a0873480d, entries=150, sequenceid=16, filesize=11.7 K 2024-11-07T09:37:38,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/c5d0b0e8a858466a8232572debb86dfd as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/c5d0b0e8a858466a8232572debb86dfd 2024-11-07T09:37:38,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/c5d0b0e8a858466a8232572debb86dfd, entries=150, sequenceid=16, filesize=11.7 K 2024-11-07T09:37:38,437 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 7f7127bc07c9cd4d3f409040fbdd89d3 in 541ms, sequenceid=16, compaction requested=false 2024-11-07T09:37:38,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7f7127bc07c9cd4d3f409040fbdd89d3: 2024-11-07T09:37:38,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-07T09:37:38,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:38,497 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 7f7127bc07c9cd4d3f409040fbdd89d3 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T09:37:38,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107ad6b3289aae943299bd3a86355cfcc8b_7f7127bc07c9cd4d3f409040fbdd89d3 is 50, key is test_row_0/A:col10/1730972257944/Put/seqid=0 2024-11-07T09:37:38,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-07T09:37:38,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742207_1383 (size=12154) 2024-11-07T09:37:38,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:38,529 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107ad6b3289aae943299bd3a86355cfcc8b_7f7127bc07c9cd4d3f409040fbdd89d3 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107ad6b3289aae943299bd3a86355cfcc8b_7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:38,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/4b041091f08043198cb88ed817b01bde, store: [table=TestAcidGuarantees family=A region=7f7127bc07c9cd4d3f409040fbdd89d3] 2024-11-07T09:37:38,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/4b041091f08043198cb88ed817b01bde is 175, key is test_row_0/A:col10/1730972257944/Put/seqid=0 2024-11-07T09:37:38,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742208_1384 (size=30955) 2024-11-07T09:37:38,562 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/4b041091f08043198cb88ed817b01bde 2024-11-07T09:37:38,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. as already flushing 2024-11-07T09:37:38,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:38,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/8177ad162a8947078d529b00499524f7 is 50, key is test_row_0/B:col10/1730972257944/Put/seqid=0 2024-11-07T09:37:38,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742209_1385 (size=12001) 2024-11-07T09:37:38,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:38,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49116 deadline: 1730972318593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:38,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:38,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:38,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:38,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:38,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49162 deadline: 1730972318594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:38,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:38,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:38,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:38,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:38,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49086 deadline: 1730972318596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:38,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:38,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:38,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:38,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:38,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49148 deadline: 1730972318597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:38,612 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:38,612 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:38,612 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:38,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:38,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49098 deadline: 1730972318610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:38,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:38,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:38,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:38,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:38,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49116 deadline: 1730972318713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:38,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:38,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:38,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:38,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49162 deadline: 1730972318713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:38,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:38,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:38,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:38,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:38,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:38,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49086 deadline: 1730972318723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:38,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:38,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:38,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49148 deadline: 1730972318723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:38,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:38,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:38,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:38,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:38,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:38,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:38,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49098 deadline: 1730972318733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:38,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:38,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:38,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:38,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:38,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49116 deadline: 1730972318922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:38,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:38,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:38,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:38,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:38,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49162 deadline: 1730972318922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:38,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:38,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:38,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:38,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:38,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49086 deadline: 1730972318933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:38,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:38,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:38,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:38,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:38,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49148 deadline: 1730972318933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:38,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:38,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:38,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:38,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:38,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49098 deadline: 1730972318953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:38,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:38,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:38,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:38,995 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/8177ad162a8947078d529b00499524f7 2024-11-07T09:37:39,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/bb491638daf2457b8c35e2b3822daea2 is 50, key is test_row_0/C:col10/1730972257944/Put/seqid=0 2024-11-07T09:37:39,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742210_1386 (size=12001) 2024-11-07T09:37:39,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-07T09:37:39,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:39,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49116 deadline: 1730972319233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:39,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:39,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49162 deadline: 1730972319233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:39,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:39,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:39,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:39,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:39,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:39,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:39,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:39,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49086 deadline: 1730972319253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:39,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:39,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:39,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:39,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:39,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49148 deadline: 1730972319253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:39,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:39,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:39,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:39,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:39,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49098 deadline: 1730972319273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:39,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:39,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:39,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:39,407 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/bb491638daf2457b8c35e2b3822daea2 2024-11-07T09:37:39,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/4b041091f08043198cb88ed817b01bde as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/4b041091f08043198cb88ed817b01bde 2024-11-07T09:37:39,415 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/4b041091f08043198cb88ed817b01bde, entries=150, sequenceid=40, filesize=30.2 K 2024-11-07T09:37:39,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/8177ad162a8947078d529b00499524f7 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/8177ad162a8947078d529b00499524f7 2024-11-07T09:37:39,420 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/8177ad162a8947078d529b00499524f7, entries=150, sequenceid=40, filesize=11.7 K 2024-11-07T09:37:39,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/bb491638daf2457b8c35e2b3822daea2 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/bb491638daf2457b8c35e2b3822daea2 2024-11-07T09:37:39,423 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/bb491638daf2457b8c35e2b3822daea2, entries=150, sequenceid=40, filesize=11.7 K 2024-11-07T09:37:39,424 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 7f7127bc07c9cd4d3f409040fbdd89d3 in 927ms, sequenceid=40, compaction requested=false 2024-11-07T09:37:39,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 7f7127bc07c9cd4d3f409040fbdd89d3: 2024-11-07T09:37:39,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:39,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-07T09:37:39,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-07T09:37:39,427 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-07T09:37:39,428 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5410 sec 2024-11-07T09:37:39,429 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.5480 sec 2024-11-07T09:37:39,533 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T09:37:39,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:39,761 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:37:39,761 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7f7127bc07c9cd4d3f409040fbdd89d3 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-07T09:37:39,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107588e7610275943e3afa7e4247f585e54_7f7127bc07c9cd4d3f409040fbdd89d3 is 50, key is test_row_0/A:col10/1730972259759/Put/seqid=0 2024-11-07T09:37:39,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742211_1387 (size=12154) 2024-11-07T09:37:39,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:39,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49086 deadline: 1730972319804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:39,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:39,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:39,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49148 deadline: 1730972319810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:39,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:39,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:39,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:39,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:39,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:39,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:39,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:39,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49162 deadline: 1730972319811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:39,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49116 deadline: 1730972319812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:39,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:39,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:39,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:39,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:39,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49098 deadline: 1730972319817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:39,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:39,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:39,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:39,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:39,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:39,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:39,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:39,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49086 deadline: 1730972319923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:39,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:39,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49148 deadline: 1730972319923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:39,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:39,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:39,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:39,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:39,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:39,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:39,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:39,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49162 deadline: 1730972319933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:39,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:39,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:39,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:39,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:39,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49116 deadline: 1730972319933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:39,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:39,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:39,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:39,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49098 deadline: 1730972319933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:39,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:39,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:39,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:39,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:40,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-07T09:37:40,044 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:37:40,044 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T09:37:40,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-07T09:37:40,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-07T09:37:40,046 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T09:37:40,046 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T09:37:40,046 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T09:37:40,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:40,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49148 deadline: 1730972320143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:40,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:40,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:40,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49086 deadline: 1730972320143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:40,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:40,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:40,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:40,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:40,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:40,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-07T09:37:40,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:40,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49162 deadline: 1730972320152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:40,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:40,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:40,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49116 deadline: 1730972320153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:40,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:40,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:40,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:40,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:40,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:40,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:40,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49098 deadline: 1730972320153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:40,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:40,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:40,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:40,177 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:40,181 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107588e7610275943e3afa7e4247f585e54_7f7127bc07c9cd4d3f409040fbdd89d3 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107588e7610275943e3afa7e4247f585e54_7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:40,182 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/479057baa25541a19a663c98d1456544, store: [table=TestAcidGuarantees family=A region=7f7127bc07c9cd4d3f409040fbdd89d3] 2024-11-07T09:37:40,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/479057baa25541a19a663c98d1456544 is 175, key is test_row_0/A:col10/1730972259759/Put/seqid=0 2024-11-07T09:37:40,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742212_1388 (size=30955) 2024-11-07T09:37:40,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-07T09:37:40,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:40,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. as already flushing 2024-11-07T09:37:40,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:40,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:40,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:40,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:40,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-07T09:37:40,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:40,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. as already flushing 2024-11-07T09:37:40,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:40,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:40,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:40,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:40,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-07T09:37:40,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:40,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49148 deadline: 1730972320463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:40,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:40,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:40,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:40,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49086 deadline: 1730972320463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:40,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:40,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:40,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:40,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:40,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49162 deadline: 1730972320472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:40,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:40,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:40,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:40,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:40,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:40,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49116 deadline: 1730972320472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:40,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:40,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:40,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:40,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:40,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49098 deadline: 1730972320473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:40,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:40,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:40,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:40,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-07T09:37:40,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:40,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. as already flushing 2024-11-07T09:37:40,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:40,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:40,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:40,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:40,587 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/479057baa25541a19a663c98d1456544 2024-11-07T09:37:40,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/d8c9a79ac6f14c8abdfffdcc48e6993e is 50, key is test_row_0/B:col10/1730972259759/Put/seqid=0 2024-11-07T09:37:40,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742213_1389 (size=12001) 2024-11-07T09:37:40,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-07T09:37:40,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:40,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. as already flushing 2024-11-07T09:37:40,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:40,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:40,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:40,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:40,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-07T09:37:40,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-07T09:37:40,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:40,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. as already flushing 2024-11-07T09:37:40,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:40,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:40,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:40,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:40,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-07T09:37:40,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:40,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. as already flushing 2024-11-07T09:37:40,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:40,963 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:40,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:40,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:40,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:40,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49148 deadline: 1730972320982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:40,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:40,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:40,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49086 deadline: 1730972320982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:40,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:40,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:40,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:40,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:40,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49162 deadline: 1730972320982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:40,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:40,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:40,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:40,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:40,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49116 deadline: 1730972320983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:40,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:40,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:40,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:40,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:40,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:40,998 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/d8c9a79ac6f14c8abdfffdcc48e6993e 2024-11-07T09:37:41,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/efa6f537a1ff47cda4aa0fe7a11bb63c is 50, key is test_row_0/C:col10/1730972259759/Put/seqid=0 2024-11-07T09:37:41,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:41,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49098 deadline: 1730972320993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:41,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:41,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:41,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:41,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742214_1390 (size=12001) 2024-11-07T09:37:41,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-07T09:37:41,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:41,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. as already flushing 2024-11-07T09:37:41,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:41,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:41,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:41,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:41,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-07T09:37:41,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-07T09:37:41,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:41,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. as already flushing 2024-11-07T09:37:41,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:41,269 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:41,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:41,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:41,409 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/efa6f537a1ff47cda4aa0fe7a11bb63c 2024-11-07T09:37:41,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/479057baa25541a19a663c98d1456544 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/479057baa25541a19a663c98d1456544 2024-11-07T09:37:41,416 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/479057baa25541a19a663c98d1456544, entries=150, sequenceid=54, filesize=30.2 K 2024-11-07T09:37:41,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/d8c9a79ac6f14c8abdfffdcc48e6993e as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/d8c9a79ac6f14c8abdfffdcc48e6993e 2024-11-07T09:37:41,420 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/d8c9a79ac6f14c8abdfffdcc48e6993e, entries=150, sequenceid=54, filesize=11.7 K 2024-11-07T09:37:41,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/efa6f537a1ff47cda4aa0fe7a11bb63c as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/efa6f537a1ff47cda4aa0fe7a11bb63c 2024-11-07T09:37:41,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-07T09:37:41,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:41,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. as already flushing 2024-11-07T09:37:41,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:41,422 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:41,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:41,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:41,424 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/efa6f537a1ff47cda4aa0fe7a11bb63c, entries=150, sequenceid=54, filesize=11.7 K 2024-11-07T09:37:41,425 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 7f7127bc07c9cd4d3f409040fbdd89d3 in 1664ms, sequenceid=54, compaction requested=true 2024-11-07T09:37:41,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7f7127bc07c9cd4d3f409040fbdd89d3: 2024-11-07T09:37:41,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f7127bc07c9cd4d3f409040fbdd89d3:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T09:37:41,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:41,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f7127bc07c9cd4d3f409040fbdd89d3:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T09:37:41,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:41,425 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:37:41,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f7127bc07c9cd4d3f409040fbdd89d3:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:37:41,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:37:41,425 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:37:41,426 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:37:41,426 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 7f7127bc07c9cd4d3f409040fbdd89d3/A is initiating minor compaction (all files) 2024-11-07T09:37:41,426 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:37:41,426 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7f7127bc07c9cd4d3f409040fbdd89d3/A in TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:41,426 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 7f7127bc07c9cd4d3f409040fbdd89d3/B is initiating minor compaction (all files) 2024-11-07T09:37:41,426 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7f7127bc07c9cd4d3f409040fbdd89d3/B in TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:41,426 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/0d4b997726f14c8396e95dfa0ff94968, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/4b041091f08043198cb88ed817b01bde, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/479057baa25541a19a663c98d1456544] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp, totalSize=107.5 K 2024-11-07T09:37:41,426 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/effc86c3e9914434b79bc59a0873480d, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/8177ad162a8947078d529b00499524f7, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/d8c9a79ac6f14c8abdfffdcc48e6993e] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp, totalSize=35.2 K 2024-11-07T09:37:41,426 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:41,426 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. files: [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/0d4b997726f14c8396e95dfa0ff94968, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/4b041091f08043198cb88ed817b01bde, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/479057baa25541a19a663c98d1456544] 2024-11-07T09:37:41,427 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting effc86c3e9914434b79bc59a0873480d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1730972257894 2024-11-07T09:37:41,427 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d4b997726f14c8396e95dfa0ff94968, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1730972257890 2024-11-07T09:37:41,427 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b041091f08043198cb88ed817b01bde, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1730972257932 2024-11-07T09:37:41,427 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 8177ad162a8947078d529b00499524f7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1730972257932 2024-11-07T09:37:41,427 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 479057baa25541a19a663c98d1456544, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1730972258589 2024-11-07T09:37:41,427 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting d8c9a79ac6f14c8abdfffdcc48e6993e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1730972258589 2024-11-07T09:37:41,432 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7f7127bc07c9cd4d3f409040fbdd89d3] 2024-11-07T09:37:41,434 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7f7127bc07c9cd4d3f409040fbdd89d3#B#compaction#338 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:37:41,434 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/77381fcde3234fbdae705254816a71ed is 50, key is test_row_0/B:col10/1730972259759/Put/seqid=0 2024-11-07T09:37:41,434 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411079a3c52e99783420b8960f46dfbbf0baa_7f7127bc07c9cd4d3f409040fbdd89d3 store=[table=TestAcidGuarantees family=A region=7f7127bc07c9cd4d3f409040fbdd89d3] 2024-11-07T09:37:41,436 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411079a3c52e99783420b8960f46dfbbf0baa_7f7127bc07c9cd4d3f409040fbdd89d3, store=[table=TestAcidGuarantees family=A region=7f7127bc07c9cd4d3f409040fbdd89d3] 2024-11-07T09:37:41,436 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411079a3c52e99783420b8960f46dfbbf0baa_7f7127bc07c9cd4d3f409040fbdd89d3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7f7127bc07c9cd4d3f409040fbdd89d3] 2024-11-07T09:37:41,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742215_1391 (size=12104) 2024-11-07T09:37:41,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742216_1392 (size=4469) 2024-11-07T09:37:41,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-07T09:37:41,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:41,575 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 7f7127bc07c9cd4d3f409040fbdd89d3 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-07T09:37:41,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107d38388ca23734112ac9357623ff668ee_7f7127bc07c9cd4d3f409040fbdd89d3 is 50, key is test_row_0/A:col10/1730972259809/Put/seqid=0 2024-11-07T09:37:41,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742217_1393 (size=12154) 2024-11-07T09:37:41,850 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/77381fcde3234fbdae705254816a71ed as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/77381fcde3234fbdae705254816a71ed 2024-11-07T09:37:41,850 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7f7127bc07c9cd4d3f409040fbdd89d3#A#compaction#339 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:37:41,850 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/ebb241d7d3ad46aaac1993829f6e77f1 is 175, key is test_row_0/A:col10/1730972259759/Put/seqid=0 2024-11-07T09:37:41,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742218_1394 (size=31058) 2024-11-07T09:37:41,855 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7f7127bc07c9cd4d3f409040fbdd89d3/B of 7f7127bc07c9cd4d3f409040fbdd89d3 into 77381fcde3234fbdae705254816a71ed(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:37:41,855 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7f7127bc07c9cd4d3f409040fbdd89d3: 2024-11-07T09:37:41,855 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., storeName=7f7127bc07c9cd4d3f409040fbdd89d3/B, priority=13, startTime=1730972261425; duration=0sec 2024-11-07T09:37:41,855 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:37:41,855 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f7127bc07c9cd4d3f409040fbdd89d3:B 2024-11-07T09:37:41,855 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:37:41,856 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:37:41,856 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 7f7127bc07c9cd4d3f409040fbdd89d3/C is initiating minor compaction (all files) 2024-11-07T09:37:41,856 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7f7127bc07c9cd4d3f409040fbdd89d3/C in TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:41,856 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/c5d0b0e8a858466a8232572debb86dfd, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/bb491638daf2457b8c35e2b3822daea2, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/efa6f537a1ff47cda4aa0fe7a11bb63c] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp, totalSize=35.2 K 2024-11-07T09:37:41,856 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting c5d0b0e8a858466a8232572debb86dfd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1730972257894 2024-11-07T09:37:41,857 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting bb491638daf2457b8c35e2b3822daea2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1730972257932 2024-11-07T09:37:41,859 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting efa6f537a1ff47cda4aa0fe7a11bb63c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1730972258589 2024-11-07T09:37:41,864 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7f7127bc07c9cd4d3f409040fbdd89d3#C#compaction#341 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:37:41,865 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/88bd9057ff244159abee084c8557ee1b is 50, key is test_row_0/C:col10/1730972259759/Put/seqid=0 2024-11-07T09:37:41,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742219_1395 (size=12104) 2024-11-07T09:37:41,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:41,991 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107d38388ca23734112ac9357623ff668ee_7f7127bc07c9cd4d3f409040fbdd89d3 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107d38388ca23734112ac9357623ff668ee_7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:41,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/f370f044157f4e4bb8a454445644665f, store: [table=TestAcidGuarantees family=A region=7f7127bc07c9cd4d3f409040fbdd89d3] 2024-11-07T09:37:41,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/f370f044157f4e4bb8a454445644665f is 175, key is test_row_0/A:col10/1730972259809/Put/seqid=0 2024-11-07T09:37:41,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742220_1396 (size=30955) 2024-11-07T09:37:42,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:42,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. as already flushing 2024-11-07T09:37:42,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:42,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49162 deadline: 1730972322023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:42,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:42,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49098 deadline: 1730972322024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:42,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:42,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:42,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49116 deadline: 1730972322025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:42,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:42,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:42,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:42,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:42,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:42,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:42,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:42,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49148 deadline: 1730972322027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:42,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:42,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:42,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:42,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:42,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:42,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:42,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49086 deadline: 1730972322028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:42,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:42,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:42,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:42,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:42,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49162 deadline: 1730972322143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:42,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:42,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49098 deadline: 1730972322143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:42,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:42,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:42,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:42,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:42,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:42,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49116 deadline: 1730972322143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:42,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:42,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:42,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:42,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:42,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:42,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:42,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49148 deadline: 1730972322143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:42,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:42,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:42,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:42,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:42,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49086 deadline: 1730972322143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:42,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:42,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:42,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:42,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-07T09:37:42,259 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/ebb241d7d3ad46aaac1993829f6e77f1 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/ebb241d7d3ad46aaac1993829f6e77f1 2024-11-07T09:37:42,263 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7f7127bc07c9cd4d3f409040fbdd89d3/A of 7f7127bc07c9cd4d3f409040fbdd89d3 into ebb241d7d3ad46aaac1993829f6e77f1(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:37:42,263 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7f7127bc07c9cd4d3f409040fbdd89d3: 2024-11-07T09:37:42,263 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., storeName=7f7127bc07c9cd4d3f409040fbdd89d3/A, priority=13, startTime=1730972261425; duration=0sec 2024-11-07T09:37:42,263 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:42,263 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f7127bc07c9cd4d3f409040fbdd89d3:A 2024-11-07T09:37:42,272 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/88bd9057ff244159abee084c8557ee1b as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/88bd9057ff244159abee084c8557ee1b 2024-11-07T09:37:42,276 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7f7127bc07c9cd4d3f409040fbdd89d3/C of 7f7127bc07c9cd4d3f409040fbdd89d3 into 88bd9057ff244159abee084c8557ee1b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:37:42,276 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7f7127bc07c9cd4d3f409040fbdd89d3: 2024-11-07T09:37:42,276 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., storeName=7f7127bc07c9cd4d3f409040fbdd89d3/C, priority=13, startTime=1730972261425; duration=0sec 2024-11-07T09:37:42,276 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:42,276 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f7127bc07c9cd4d3f409040fbdd89d3:C 2024-11-07T09:37:42,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:42,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49162 deadline: 1730972322363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:42,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:42,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49098 deadline: 1730972322363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:42,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:42,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:42,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:42,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:42,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:42,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49116 deadline: 1730972322363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:42,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:42,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:42,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:42,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:42,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49148 deadline: 1730972322363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:42,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:42,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:42,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:42,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:42,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49086 deadline: 1730972322363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:42,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:42,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:42,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:42,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:42,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:42,397 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=76, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/f370f044157f4e4bb8a454445644665f 2024-11-07T09:37:42,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/35c0cd9cb5b245fdb779b76abddfc436 is 50, key is test_row_0/B:col10/1730972259809/Put/seqid=0 2024-11-07T09:37:42,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742221_1397 (size=12001) 2024-11-07T09:37:42,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:42,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49162 deadline: 1730972322683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:42,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:42,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:42,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:42,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49098 deadline: 1730972322683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:42,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:42,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:42,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49116 deadline: 1730972322683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:42,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:42,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:42,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:42,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:42,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49148 deadline: 1730972322683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:42,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:42,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:42,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:42,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:42,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:42,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:42,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49086 deadline: 1730972322683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:42,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:42,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:42,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:42,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:42,808 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/35c0cd9cb5b245fdb779b76abddfc436 2024-11-07T09:37:42,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/2f58d38a34d64cd9891eadc9f641e5e0 is 50, key is test_row_0/C:col10/1730972259809/Put/seqid=0 2024-11-07T09:37:42,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742222_1398 (size=12001) 2024-11-07T09:37:43,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:43,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49162 deadline: 1730972323203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:43,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:43,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49098 deadline: 1730972323203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:43,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:43,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:43,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:43,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:43,212 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:43,212 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:43,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49116 deadline: 1730972323203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:43,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:43,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:43,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:43,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:43,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:43,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49148 deadline: 1730972323203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:43,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:43,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:43,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:43,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:43,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49086 deadline: 1730972323203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:43,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:43,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:43,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:43,220 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/2f58d38a34d64cd9891eadc9f641e5e0 2024-11-07T09:37:43,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/f370f044157f4e4bb8a454445644665f as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/f370f044157f4e4bb8a454445644665f 2024-11-07T09:37:43,227 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/f370f044157f4e4bb8a454445644665f, entries=150, sequenceid=76, filesize=30.2 K 2024-11-07T09:37:43,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/35c0cd9cb5b245fdb779b76abddfc436 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/35c0cd9cb5b245fdb779b76abddfc436 2024-11-07T09:37:43,231 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/35c0cd9cb5b245fdb779b76abddfc436, entries=150, sequenceid=76, filesize=11.7 K 2024-11-07T09:37:43,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/2f58d38a34d64cd9891eadc9f641e5e0 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/2f58d38a34d64cd9891eadc9f641e5e0 2024-11-07T09:37:43,235 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/2f58d38a34d64cd9891eadc9f641e5e0, entries=150, sequenceid=76, filesize=11.7 K 2024-11-07T09:37:43,236 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 7f7127bc07c9cd4d3f409040fbdd89d3 in 1661ms, sequenceid=76, compaction requested=false 2024-11-07T09:37:43,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 7f7127bc07c9cd4d3f409040fbdd89d3: 2024-11-07T09:37:43,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:43,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-07T09:37:43,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-07T09:37:43,239 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-07T09:37:43,239 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1920 sec 2024-11-07T09:37:43,241 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 3.1960 sec 2024-11-07T09:37:44,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-07T09:37:44,213 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:37:44,213 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T09:37:44,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-07T09:37:44,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-07T09:37:44,215 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T09:37:44,216 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T09:37:44,216 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T09:37:44,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:44,229 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:37:44,229 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7f7127bc07c9cd4d3f409040fbdd89d3 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-07T09:37:44,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411076ac869c535754f98887c2675e7563ed5_7f7127bc07c9cd4d3f409040fbdd89d3 is 50, key is test_row_0/A:col10/1730972262022/Put/seqid=0 2024-11-07T09:37:44,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742223_1399 (size=17034) 2024-11-07T09:37:44,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:44,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49148 deadline: 1730972324259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:44,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:44,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:44,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49162 deadline: 1730972324260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:44,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:44,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:44,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:44,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49098 deadline: 1730972324261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:44,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:44,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:44,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:44,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:44,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:44,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:44,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:44,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49086 deadline: 1730972324263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:44,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:44,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:44,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:44,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:44,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49116 deadline: 1730972324274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:44,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:44,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:44,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:44,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-07T09:37:44,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-07T09:37:44,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:44,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. as already flushing 2024-11-07T09:37:44,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:44,368 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:44,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:44,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:44,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:44,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49148 deadline: 1730972324383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:44,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:44,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:44,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:44,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49162 deadline: 1730972324383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:44,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:44,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:44,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:44,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:44,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:44,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49098 deadline: 1730972324383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:44,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:44,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:44,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:44,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49086 deadline: 1730972324383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:44,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:44,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:44,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:44,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:44,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:44,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49116 deadline: 1730972324393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:44,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:44,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:44,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:44,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-07T09:37:44,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:44,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. as already flushing 2024-11-07T09:37:44,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:44,522 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:44,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:44,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:44,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-07T09:37:44,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:44,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49148 deadline: 1730972324603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:44,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:44,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:44,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:44,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:44,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49162 deadline: 1730972324603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:44,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:44,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:44,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:44,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:44,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49098 deadline: 1730972324603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:44,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:44,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:44,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:44,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49086 deadline: 1730972324603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:44,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:44,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:44,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:44,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:44,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:44,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49116 deadline: 1730972324612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:44,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:44,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:44,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:44,645 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:44,648 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411076ac869c535754f98887c2675e7563ed5_7f7127bc07c9cd4d3f409040fbdd89d3 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411076ac869c535754f98887c2675e7563ed5_7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:44,649 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/4f4c2857859c4d98beb897acca385438, store: [table=TestAcidGuarantees family=A region=7f7127bc07c9cd4d3f409040fbdd89d3] 2024-11-07T09:37:44,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/4f4c2857859c4d98beb897acca385438 is 175, key is test_row_0/A:col10/1730972262022/Put/seqid=0 2024-11-07T09:37:44,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742224_1400 (size=48139) 2024-11-07T09:37:44,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-07T09:37:44,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:44,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. as already flushing 2024-11-07T09:37:44,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:44,675 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:44,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:44,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:44,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-07T09:37:44,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:44,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. as already flushing 2024-11-07T09:37:44,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:44,828 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:44,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:44,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:44,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-07T09:37:44,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:44,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49148 deadline: 1730972324923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:44,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:44,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:44,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49162 deadline: 1730972324923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:44,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:44,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:44,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:44,933 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:44,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49098 deadline: 1730972324923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:44,933 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:44,933 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:44,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:44,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:44,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:44,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49086 deadline: 1730972324923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:44,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:44,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:44,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:44,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:44,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:44,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49116 deadline: 1730972324933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:44,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:44,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:44,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:44,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-07T09:37:44,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:44,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. as already flushing 2024-11-07T09:37:44,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:44,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:44,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:44,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:45,054 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/4f4c2857859c4d98beb897acca385438 2024-11-07T09:37:45,061 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/76f51d0333b14f1c9d929b2f58befe22 is 50, key is test_row_0/B:col10/1730972262022/Put/seqid=0 2024-11-07T09:37:45,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742225_1401 (size=12001) 2024-11-07T09:37:45,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-07T09:37:45,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:45,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. as already flushing 2024-11-07T09:37:45,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:45,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:45,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:45,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:45,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-07T09:37:45,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:45,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. as already flushing 2024-11-07T09:37:45,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:45,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:45,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:45,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:45,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-07T09:37:45,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-07T09:37:45,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:45,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. as already flushing 2024-11-07T09:37:45,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:45,439 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:45,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:45,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:45,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:45,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49148 deadline: 1730972325443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:45,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:45,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49162 deadline: 1730972325443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:45,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:45,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:45,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:45,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:45,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49098 deadline: 1730972325443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:45,454 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:45,454 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:45,454 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:45,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:45,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:45,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49116 deadline: 1730972325444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:45,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:45,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:45,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:45,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:45,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:45,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:45,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49086 deadline: 1730972325444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:45,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:45,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:45,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:45,465 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/76f51d0333b14f1c9d929b2f58befe22 2024-11-07T09:37:45,472 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/7e98b0fbd64d48c78b3ed54efa7daeb0 is 50, key is test_row_0/C:col10/1730972262022/Put/seqid=0 2024-11-07T09:37:45,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742226_1402 (size=12001) 2024-11-07T09:37:45,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-07T09:37:45,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:45,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. as already flushing 2024-11-07T09:37:45,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:45,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:45,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:45,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:45,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-07T09:37:45,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:45,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. as already flushing 2024-11-07T09:37:45,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:45,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:45,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:45,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:45,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/7e98b0fbd64d48c78b3ed54efa7daeb0 2024-11-07T09:37:45,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/4f4c2857859c4d98beb897acca385438 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/4f4c2857859c4d98beb897acca385438 2024-11-07T09:37:45,886 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/4f4c2857859c4d98beb897acca385438, entries=250, sequenceid=95, filesize=47.0 K 2024-11-07T09:37:45,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/76f51d0333b14f1c9d929b2f58befe22 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/76f51d0333b14f1c9d929b2f58befe22 2024-11-07T09:37:45,890 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/76f51d0333b14f1c9d929b2f58befe22, entries=150, sequenceid=95, filesize=11.7 K 2024-11-07T09:37:45,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/7e98b0fbd64d48c78b3ed54efa7daeb0 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/7e98b0fbd64d48c78b3ed54efa7daeb0 2024-11-07T09:37:45,895 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/7e98b0fbd64d48c78b3ed54efa7daeb0, entries=150, sequenceid=95, filesize=11.7 K 2024-11-07T09:37:45,895 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 7f7127bc07c9cd4d3f409040fbdd89d3 in 1666ms, sequenceid=95, compaction requested=true 2024-11-07T09:37:45,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7f7127bc07c9cd4d3f409040fbdd89d3: 2024-11-07T09:37:45,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f7127bc07c9cd4d3f409040fbdd89d3:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T09:37:45,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:45,896 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:37:45,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f7127bc07c9cd4d3f409040fbdd89d3:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T09:37:45,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:45,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f7127bc07c9cd4d3f409040fbdd89d3:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:37:45,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:37:45,896 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:37:45,896 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110152 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:37:45,896 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 7f7127bc07c9cd4d3f409040fbdd89d3/A is initiating minor compaction (all files) 2024-11-07T09:37:45,896 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:37:45,896 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7f7127bc07c9cd4d3f409040fbdd89d3/A in TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:45,896 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 7f7127bc07c9cd4d3f409040fbdd89d3/B is initiating minor compaction (all files) 2024-11-07T09:37:45,896 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/ebb241d7d3ad46aaac1993829f6e77f1, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/f370f044157f4e4bb8a454445644665f, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/4f4c2857859c4d98beb897acca385438] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp, totalSize=107.6 K 2024-11-07T09:37:45,896 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7f7127bc07c9cd4d3f409040fbdd89d3/B in TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:45,897 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:45,897 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. files: [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/ebb241d7d3ad46aaac1993829f6e77f1, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/f370f044157f4e4bb8a454445644665f, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/4f4c2857859c4d98beb897acca385438] 2024-11-07T09:37:45,897 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/77381fcde3234fbdae705254816a71ed, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/35c0cd9cb5b245fdb779b76abddfc436, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/76f51d0333b14f1c9d929b2f58befe22] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp, totalSize=35.3 K 2024-11-07T09:37:45,897 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebb241d7d3ad46aaac1993829f6e77f1, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1730972258589 2024-11-07T09:37:45,897 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 77381fcde3234fbdae705254816a71ed, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1730972258589 2024-11-07T09:37:45,897 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 35c0cd9cb5b245fdb779b76abddfc436, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1730972259798 2024-11-07T09:37:45,897 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting f370f044157f4e4bb8a454445644665f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1730972259798 2024-11-07T09:37:45,897 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f4c2857859c4d98beb897acca385438, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1730972262022 2024-11-07T09:37:45,897 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 76f51d0333b14f1c9d929b2f58befe22, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1730972262022 2024-11-07T09:37:45,898 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-07T09:37:45,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:45,899 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 7f7127bc07c9cd4d3f409040fbdd89d3 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-07T09:37:45,915 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7f7127bc07c9cd4d3f409040fbdd89d3] 2024-11-07T09:37:45,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110774b88e95c6b74adc809b5a12848d390c_7f7127bc07c9cd4d3f409040fbdd89d3 is 50, key is test_row_0/A:col10/1730972264258/Put/seqid=0 2024-11-07T09:37:45,917 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7f7127bc07c9cd4d3f409040fbdd89d3#B#compaction#349 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:37:45,918 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/a6fbff66161c4796a4c167a555da83c9 is 50, key is test_row_0/B:col10/1730972262022/Put/seqid=0 2024-11-07T09:37:45,921 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107179a593e97724704a0dea5e08e64d390_7f7127bc07c9cd4d3f409040fbdd89d3 store=[table=TestAcidGuarantees family=A region=7f7127bc07c9cd4d3f409040fbdd89d3] 2024-11-07T09:37:45,923 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107179a593e97724704a0dea5e08e64d390_7f7127bc07c9cd4d3f409040fbdd89d3, store=[table=TestAcidGuarantees family=A region=7f7127bc07c9cd4d3f409040fbdd89d3] 2024-11-07T09:37:45,923 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107179a593e97724704a0dea5e08e64d390_7f7127bc07c9cd4d3f409040fbdd89d3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7f7127bc07c9cd4d3f409040fbdd89d3] 2024-11-07T09:37:45,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742227_1403 (size=12154) 2024-11-07T09:37:45,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:45,937 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110774b88e95c6b74adc809b5a12848d390c_7f7127bc07c9cd4d3f409040fbdd89d3 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110774b88e95c6b74adc809b5a12848d390c_7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:45,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742228_1404 (size=12207) 2024-11-07T09:37:45,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/2d2b4eed4ca147e4a30c73ece6202340, store: [table=TestAcidGuarantees family=A region=7f7127bc07c9cd4d3f409040fbdd89d3] 2024-11-07T09:37:45,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/2d2b4eed4ca147e4a30c73ece6202340 is 175, key is test_row_0/A:col10/1730972264258/Put/seqid=0 2024-11-07T09:37:45,947 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/a6fbff66161c4796a4c167a555da83c9 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/a6fbff66161c4796a4c167a555da83c9 2024-11-07T09:37:45,956 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7f7127bc07c9cd4d3f409040fbdd89d3/B of 7f7127bc07c9cd4d3f409040fbdd89d3 into a6fbff66161c4796a4c167a555da83c9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:37:45,956 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7f7127bc07c9cd4d3f409040fbdd89d3: 2024-11-07T09:37:45,956 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., storeName=7f7127bc07c9cd4d3f409040fbdd89d3/B, priority=13, startTime=1730972265896; duration=0sec 2024-11-07T09:37:45,956 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:37:45,956 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f7127bc07c9cd4d3f409040fbdd89d3:B 2024-11-07T09:37:45,956 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:37:45,957 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:37:45,957 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 7f7127bc07c9cd4d3f409040fbdd89d3/C is initiating minor compaction (all files) 2024-11-07T09:37:45,957 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7f7127bc07c9cd4d3f409040fbdd89d3/C in TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:45,957 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/88bd9057ff244159abee084c8557ee1b, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/2f58d38a34d64cd9891eadc9f641e5e0, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/7e98b0fbd64d48c78b3ed54efa7daeb0] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp, totalSize=35.3 K 2024-11-07T09:37:45,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742229_1405 (size=4469) 2024-11-07T09:37:45,958 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 88bd9057ff244159abee084c8557ee1b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1730972258589 2024-11-07T09:37:45,958 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f58d38a34d64cd9891eadc9f641e5e0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1730972259798 2024-11-07T09:37:45,959 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7f7127bc07c9cd4d3f409040fbdd89d3#A#compaction#348 average throughput is 0.56 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:37:45,959 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e98b0fbd64d48c78b3ed54efa7daeb0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1730972262022 2024-11-07T09:37:45,960 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/e45294a9e04d4af2a8fc5c1cf47d781e is 175, key is test_row_0/A:col10/1730972262022/Put/seqid=0 2024-11-07T09:37:45,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742230_1406 (size=30955) 2024-11-07T09:37:45,977 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7f7127bc07c9cd4d3f409040fbdd89d3#C#compaction#350 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:37:45,977 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/024a867da0cf4e118019c1095fe17407 is 50, key is test_row_0/C:col10/1730972262022/Put/seqid=0 2024-11-07T09:37:45,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742231_1407 (size=31161) 2024-11-07T09:37:45,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742232_1408 (size=12207) 2024-11-07T09:37:46,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-07T09:37:46,372 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=115, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/2d2b4eed4ca147e4a30c73ece6202340 2024-11-07T09:37:46,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/21a8216b215449d3aa9dd04cbb39a6ba is 50, key is test_row_0/B:col10/1730972264258/Put/seqid=0 2024-11-07T09:37:46,383 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/e45294a9e04d4af2a8fc5c1cf47d781e as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/e45294a9e04d4af2a8fc5c1cf47d781e 2024-11-07T09:37:46,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742233_1409 (size=12001) 2024-11-07T09:37:46,385 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/21a8216b215449d3aa9dd04cbb39a6ba 2024-11-07T09:37:46,391 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7f7127bc07c9cd4d3f409040fbdd89d3/A of 7f7127bc07c9cd4d3f409040fbdd89d3 into e45294a9e04d4af2a8fc5c1cf47d781e(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:37:46,391 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7f7127bc07c9cd4d3f409040fbdd89d3: 2024-11-07T09:37:46,392 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., storeName=7f7127bc07c9cd4d3f409040fbdd89d3/A, priority=13, startTime=1730972265895; duration=0sec 2024-11-07T09:37:46,392 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:46,392 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f7127bc07c9cd4d3f409040fbdd89d3:A 2024-11-07T09:37:46,397 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/024a867da0cf4e118019c1095fe17407 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/024a867da0cf4e118019c1095fe17407 2024-11-07T09:37:46,401 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7f7127bc07c9cd4d3f409040fbdd89d3/C of 7f7127bc07c9cd4d3f409040fbdd89d3 into 024a867da0cf4e118019c1095fe17407(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:37:46,401 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7f7127bc07c9cd4d3f409040fbdd89d3: 2024-11-07T09:37:46,401 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., storeName=7f7127bc07c9cd4d3f409040fbdd89d3/C, priority=13, startTime=1730972265896; duration=0sec 2024-11-07T09:37:46,401 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:46,401 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f7127bc07c9cd4d3f409040fbdd89d3:C 2024-11-07T09:37:46,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/f8753d1b00a542b889652ff83f3b7257 is 50, key is test_row_0/C:col10/1730972264258/Put/seqid=0 2024-11-07T09:37:46,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742234_1410 (size=12001) 2024-11-07T09:37:46,416 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/f8753d1b00a542b889652ff83f3b7257 2024-11-07T09:37:46,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/2d2b4eed4ca147e4a30c73ece6202340 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/2d2b4eed4ca147e4a30c73ece6202340 2024-11-07T09:37:46,427 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/2d2b4eed4ca147e4a30c73ece6202340, entries=150, sequenceid=115, filesize=30.2 K 2024-11-07T09:37:46,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/21a8216b215449d3aa9dd04cbb39a6ba as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/21a8216b215449d3aa9dd04cbb39a6ba 2024-11-07T09:37:46,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,433 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/21a8216b215449d3aa9dd04cbb39a6ba, entries=150, sequenceid=115, filesize=11.7 K 2024-11-07T09:37:46,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/f8753d1b00a542b889652ff83f3b7257 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/f8753d1b00a542b889652ff83f3b7257 2024-11-07T09:37:46,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,438 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/f8753d1b00a542b889652ff83f3b7257, entries=150, sequenceid=115, filesize=11.7 K 2024-11-07T09:37:46,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,449 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=0 B/0 for 7f7127bc07c9cd4d3f409040fbdd89d3 in 550ms, sequenceid=115, compaction requested=false 2024-11-07T09:37:46,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 7f7127bc07c9cd4d3f409040fbdd89d3: 2024-11-07T09:37:46,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:46,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-07T09:37:46,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-07T09:37:46,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,453 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-07T09:37:46,453 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2350 sec 2024-11-07T09:37:46,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,455 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 2.2400 sec 2024-11-07T09:37:46,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:46,644 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:37:46,644 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7f7127bc07c9cd4d3f409040fbdd89d3 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T09:37:46,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110724e3ac8e2f6d4228ba3c9c652866e90f_7f7127bc07c9cd4d3f409040fbdd89d3 is 50, key is test_row_0/A:col10/1730972266643/Put/seqid=0 2024-11-07T09:37:46,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742235_1411 (size=19574) 2024-11-07T09:37:46,678 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:46,684 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110724e3ac8e2f6d4228ba3c9c652866e90f_7f7127bc07c9cd4d3f409040fbdd89d3 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110724e3ac8e2f6d4228ba3c9c652866e90f_7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:46,685 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/c53bee323e764e9cad543e57e12e7389, store: [table=TestAcidGuarantees family=A region=7f7127bc07c9cd4d3f409040fbdd89d3] 2024-11-07T09:37:46,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/c53bee323e764e9cad543e57e12e7389 is 175, key is test_row_0/A:col10/1730972266643/Put/seqid=0 2024-11-07T09:37:46,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742236_1412 (size=56833) 2024-11-07T09:37:46,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:46,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49148 deadline: 1730972326703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:46,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:46,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49162 deadline: 1730972326704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:46,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:46,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:46,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:46,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:46,720 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:46,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49098 deadline: 1730972326707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:46,720 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:46,720 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:46,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:46,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:46,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:46,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:46,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49116 deadline: 1730972326708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:46,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:46,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:46,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:46,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:46,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49086 deadline: 1730972326720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:46,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:46,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:46,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:46,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:46,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49148 deadline: 1730972326823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:46,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:46,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:46,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:46,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:46,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49162 deadline: 1730972326823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:46,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:46,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:46,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:46,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:46,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49098 deadline: 1730972326823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:46,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:46,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:46,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:46,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:46,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49116 deadline: 1730972326833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:46,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:46,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:46,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:46,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:46,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49086 deadline: 1730972326843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:46,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:46,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:46,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:47,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:47,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49148 deadline: 1730972327033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:47,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:47,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:47,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:47,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:47,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49162 deadline: 1730972327035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:47,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:47,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:47,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:47,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:47,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49098 deadline: 1730972327043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:47,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:47,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:47,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:47,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:47,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49116 deadline: 1730972327053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:47,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:47,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:47,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:47,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:47,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49086 deadline: 1730972327063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:47,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:47,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:47,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:47,102 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=130, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/c53bee323e764e9cad543e57e12e7389 2024-11-07T09:37:47,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/c799eec1b6af4085bcfa558f44f51e77 is 50, key is test_row_0/B:col10/1730972266643/Put/seqid=0 2024-11-07T09:37:47,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742237_1413 (size=12101) 2024-11-07T09:37:47,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:47,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49162 deadline: 1730972327354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:47,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:47,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49148 deadline: 1730972327354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:47,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:47,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49098 deadline: 1730972327355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:47,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:47,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:47,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:47,358 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:47,358 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:47,358 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:47,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:47,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:47,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:47,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:47,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49116 deadline: 1730972327372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:47,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:47,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:47,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:47,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:47,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49086 deadline: 1730972327383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:47,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:47,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:47,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:47,518 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/c799eec1b6af4085bcfa558f44f51e77 2024-11-07T09:37:47,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/b69ad902f45749899afd5ccac22a1428 is 50, key is test_row_0/C:col10/1730972266643/Put/seqid=0 2024-11-07T09:37:47,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742238_1414 (size=12101) 2024-11-07T09:37:47,589 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/b69ad902f45749899afd5ccac22a1428 2024-11-07T09:37:47,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/c53bee323e764e9cad543e57e12e7389 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/c53bee323e764e9cad543e57e12e7389 2024-11-07T09:37:47,611 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/c53bee323e764e9cad543e57e12e7389, entries=300, sequenceid=130, filesize=55.5 K 2024-11-07T09:37:47,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/c799eec1b6af4085bcfa558f44f51e77 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/c799eec1b6af4085bcfa558f44f51e77 2024-11-07T09:37:47,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/c799eec1b6af4085bcfa558f44f51e77, entries=150, sequenceid=130, filesize=11.8 K 2024-11-07T09:37:47,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/b69ad902f45749899afd5ccac22a1428 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/b69ad902f45749899afd5ccac22a1428 2024-11-07T09:37:47,631 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/b69ad902f45749899afd5ccac22a1428, entries=150, sequenceid=130, filesize=11.8 K 2024-11-07T09:37:47,633 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 7f7127bc07c9cd4d3f409040fbdd89d3 in 988ms, sequenceid=130, compaction requested=true 2024-11-07T09:37:47,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7f7127bc07c9cd4d3f409040fbdd89d3: 2024-11-07T09:37:47,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f7127bc07c9cd4d3f409040fbdd89d3:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T09:37:47,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:47,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f7127bc07c9cd4d3f409040fbdd89d3:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T09:37:47,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:47,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f7127bc07c9cd4d3f409040fbdd89d3:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:37:47,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:37:47,633 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:37:47,633 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:37:47,635 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:37:47,635 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118949 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:37:47,635 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 7f7127bc07c9cd4d3f409040fbdd89d3/B is initiating minor compaction (all files) 2024-11-07T09:37:47,635 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 7f7127bc07c9cd4d3f409040fbdd89d3/A is initiating minor compaction (all files) 2024-11-07T09:37:47,635 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7f7127bc07c9cd4d3f409040fbdd89d3/B in TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:47,635 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7f7127bc07c9cd4d3f409040fbdd89d3/A in TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:47,635 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/a6fbff66161c4796a4c167a555da83c9, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/21a8216b215449d3aa9dd04cbb39a6ba, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/c799eec1b6af4085bcfa558f44f51e77] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp, totalSize=35.5 K 2024-11-07T09:37:47,635 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/e45294a9e04d4af2a8fc5c1cf47d781e, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/2d2b4eed4ca147e4a30c73ece6202340, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/c53bee323e764e9cad543e57e12e7389] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp, totalSize=116.2 K 2024-11-07T09:37:47,635 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:47,635 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. files: [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/e45294a9e04d4af2a8fc5c1cf47d781e, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/2d2b4eed4ca147e4a30c73ece6202340, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/c53bee323e764e9cad543e57e12e7389] 2024-11-07T09:37:47,635 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting a6fbff66161c4796a4c167a555da83c9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1730972262022 2024-11-07T09:37:47,636 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting e45294a9e04d4af2a8fc5c1cf47d781e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1730972262022 2024-11-07T09:37:47,636 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 21a8216b215449d3aa9dd04cbb39a6ba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1730972264258 2024-11-07T09:37:47,636 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d2b4eed4ca147e4a30c73ece6202340, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1730972264258 2024-11-07T09:37:47,636 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting c799eec1b6af4085bcfa558f44f51e77, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1730972266643 2024-11-07T09:37:47,636 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting c53bee323e764e9cad543e57e12e7389, keycount=300, bloomtype=ROW, size=55.5 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1730972266539 2024-11-07T09:37:47,651 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7f7127bc07c9cd4d3f409040fbdd89d3#B#compaction#356 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:37:47,652 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/2e42873a33c84082ad7079c2d11b646d is 50, key is test_row_0/B:col10/1730972266643/Put/seqid=0 2024-11-07T09:37:47,663 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7f7127bc07c9cd4d3f409040fbdd89d3] 2024-11-07T09:37:47,689 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411073958be981b76431d96b814e7525b1ef6_7f7127bc07c9cd4d3f409040fbdd89d3 store=[table=TestAcidGuarantees family=A region=7f7127bc07c9cd4d3f409040fbdd89d3] 2024-11-07T09:37:47,691 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411073958be981b76431d96b814e7525b1ef6_7f7127bc07c9cd4d3f409040fbdd89d3, store=[table=TestAcidGuarantees family=A region=7f7127bc07c9cd4d3f409040fbdd89d3] 2024-11-07T09:37:47,691 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411073958be981b76431d96b814e7525b1ef6_7f7127bc07c9cd4d3f409040fbdd89d3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7f7127bc07c9cd4d3f409040fbdd89d3] 2024-11-07T09:37:47,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742239_1415 (size=12409) 2024-11-07T09:37:47,729 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/2e42873a33c84082ad7079c2d11b646d as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/2e42873a33c84082ad7079c2d11b646d 2024-11-07T09:37:47,739 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7f7127bc07c9cd4d3f409040fbdd89d3/B of 7f7127bc07c9cd4d3f409040fbdd89d3 into 2e42873a33c84082ad7079c2d11b646d(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:37:47,739 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7f7127bc07c9cd4d3f409040fbdd89d3: 2024-11-07T09:37:47,739 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., storeName=7f7127bc07c9cd4d3f409040fbdd89d3/B, priority=13, startTime=1730972267633; duration=0sec 2024-11-07T09:37:47,739 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:37:47,739 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f7127bc07c9cd4d3f409040fbdd89d3:B 2024-11-07T09:37:47,739 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:37:47,742 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:37:47,742 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 7f7127bc07c9cd4d3f409040fbdd89d3/C is initiating minor compaction (all files) 2024-11-07T09:37:47,742 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7f7127bc07c9cd4d3f409040fbdd89d3/C in TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:47,742 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/024a867da0cf4e118019c1095fe17407, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/f8753d1b00a542b889652ff83f3b7257, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/b69ad902f45749899afd5ccac22a1428] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp, totalSize=35.5 K 2024-11-07T09:37:47,743 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 024a867da0cf4e118019c1095fe17407, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1730972262022 2024-11-07T09:37:47,744 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting f8753d1b00a542b889652ff83f3b7257, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1730972264258 2024-11-07T09:37:47,744 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting b69ad902f45749899afd5ccac22a1428, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1730972266643 2024-11-07T09:37:47,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742240_1416 (size=4469) 2024-11-07T09:37:47,768 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7f7127bc07c9cd4d3f409040fbdd89d3#C#compaction#358 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:37:47,768 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/34d4c3063d134d319370c82f642ebfe9 is 50, key is test_row_0/C:col10/1730972266643/Put/seqid=0 2024-11-07T09:37:47,772 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7f7127bc07c9cd4d3f409040fbdd89d3#A#compaction#357 average throughput is 0.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:37:47,773 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/e85fe8674a744497b88d0dfa62f96685 is 175, key is test_row_0/A:col10/1730972266643/Put/seqid=0 2024-11-07T09:37:47,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742242_1418 (size=31363) 2024-11-07T09:37:47,846 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/e85fe8674a744497b88d0dfa62f96685 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/e85fe8674a744497b88d0dfa62f96685 2024-11-07T09:37:47,852 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7f7127bc07c9cd4d3f409040fbdd89d3/A of 7f7127bc07c9cd4d3f409040fbdd89d3 into e85fe8674a744497b88d0dfa62f96685(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:37:47,852 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7f7127bc07c9cd4d3f409040fbdd89d3: 2024-11-07T09:37:47,853 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., storeName=7f7127bc07c9cd4d3f409040fbdd89d3/A, priority=13, startTime=1730972267633; duration=0sec 2024-11-07T09:37:47,853 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:47,853 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f7127bc07c9cd4d3f409040fbdd89d3:A 2024-11-07T09:37:47,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742241_1417 (size=12409) 2024-11-07T09:37:47,872 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:37:47,873 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7f7127bc07c9cd4d3f409040fbdd89d3 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T09:37:47,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:47,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107db4386e1658e41d9bbd804f3ef732ef8_7f7127bc07c9cd4d3f409040fbdd89d3 is 50, key is test_row_0/A:col10/1730972266705/Put/seqid=0 2024-11-07T09:37:47,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:47,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49162 deadline: 1730972327898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:47,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:47,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49148 deadline: 1730972327900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:47,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:47,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:47,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:47,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:47,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:47,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:47,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:47,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49116 deadline: 1730972327911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:47,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:47,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:47,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:47,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49086 deadline: 1730972327913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:47,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:47,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:47,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:47,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:47,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:47,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49098 deadline: 1730972327914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:47,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:47,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:47,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:47,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742243_1419 (size=14794) 2024-11-07T09:37:47,943 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:47,947 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107db4386e1658e41d9bbd804f3ef732ef8_7f7127bc07c9cd4d3f409040fbdd89d3 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107db4386e1658e41d9bbd804f3ef732ef8_7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:47,948 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/e0482d64ea8b462ea5262df9edee7153, store: [table=TestAcidGuarantees family=A region=7f7127bc07c9cd4d3f409040fbdd89d3] 2024-11-07T09:37:47,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/e0482d64ea8b462ea5262df9edee7153 is 175, key is test_row_0/A:col10/1730972266705/Put/seqid=0 2024-11-07T09:37:47,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742244_1420 (size=39749) 2024-11-07T09:37:48,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:48,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49162 deadline: 1730972328024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:48,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:48,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:48,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:48,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49148 deadline: 1730972328025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:48,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:48,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:48,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:48,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:48,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:48,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49116 deadline: 1730972328033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:48,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:48,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49086 deadline: 1730972328033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:48,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:48,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49098 deadline: 1730972328034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:48,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:48,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:48,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:48,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:48,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:48,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:48,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:48,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:48,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:48,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:48,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49162 deadline: 1730972328243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:48,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:48,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:48,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:48,262 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/34d4c3063d134d319370c82f642ebfe9 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/34d4c3063d134d319370c82f642ebfe9 2024-11-07T09:37:48,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:48,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49148 deadline: 1730972328254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:48,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:48,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49098 deadline: 1730972328255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:48,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:48,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:48,267 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:48,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49116 deadline: 1730972328257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:48,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:48,267 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:48,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:48,267 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:48,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:48,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49086 deadline: 1730972328257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:48,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:48,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:48,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:48,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:48,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:48,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:48,273 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7f7127bc07c9cd4d3f409040fbdd89d3/C of 7f7127bc07c9cd4d3f409040fbdd89d3 into 34d4c3063d134d319370c82f642ebfe9(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:37:48,273 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7f7127bc07c9cd4d3f409040fbdd89d3: 2024-11-07T09:37:48,273 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., storeName=7f7127bc07c9cd4d3f409040fbdd89d3/C, priority=13, startTime=1730972267633; duration=0sec 2024-11-07T09:37:48,273 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:48,273 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f7127bc07c9cd4d3f409040fbdd89d3:C 2024-11-07T09:37:48,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-07T09:37:48,385 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:37:48,386 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T09:37:48,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-07T09:37:48,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-07T09:37:48,388 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T09:37:48,389 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T09:37:48,389 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T09:37:48,394 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/e0482d64ea8b462ea5262df9edee7153 2024-11-07T09:37:48,414 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/ad0eed3e07204969a202054406042afc is 50, key is test_row_0/B:col10/1730972266705/Put/seqid=0 2024-11-07T09:37:48,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742245_1421 (size=12151) 2024-11-07T09:37:48,434 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/ad0eed3e07204969a202054406042afc 2024-11-07T09:37:48,450 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/37c7bfcefea64727bad363f9c267f95a is 50, key is test_row_0/C:col10/1730972266705/Put/seqid=0 2024-11-07T09:37:48,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742246_1422 (size=12151) 2024-11-07T09:37:48,466 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/37c7bfcefea64727bad363f9c267f95a 2024-11-07T09:37:48,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/e0482d64ea8b462ea5262df9edee7153 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/e0482d64ea8b462ea5262df9edee7153 2024-11-07T09:37:48,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/e0482d64ea8b462ea5262df9edee7153, entries=200, sequenceid=157, filesize=38.8 K 2024-11-07T09:37:48,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/ad0eed3e07204969a202054406042afc as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/ad0eed3e07204969a202054406042afc 2024-11-07T09:37:48,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/ad0eed3e07204969a202054406042afc, entries=150, sequenceid=157, filesize=11.9 K 2024-11-07T09:37:48,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/37c7bfcefea64727bad363f9c267f95a as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/37c7bfcefea64727bad363f9c267f95a 2024-11-07T09:37:48,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/37c7bfcefea64727bad363f9c267f95a, entries=150, sequenceid=157, filesize=11.9 K 2024-11-07T09:37:48,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-07T09:37:48,493 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 7f7127bc07c9cd4d3f409040fbdd89d3 in 620ms, sequenceid=157, compaction requested=false 2024-11-07T09:37:48,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7f7127bc07c9cd4d3f409040fbdd89d3: 2024-11-07T09:37:48,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-07T09:37:48,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:48,542 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 7f7127bc07c9cd4d3f409040fbdd89d3 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-07T09:37:48,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107e9b0c1c8a0614ffb876665f7971e996c_7f7127bc07c9cd4d3f409040fbdd89d3 is 50, key is test_row_0/A:col10/1730972267910/Put/seqid=0 2024-11-07T09:37:48,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:48,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. as already flushing 2024-11-07T09:37:48,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742247_1423 (size=12304) 2024-11-07T09:37:48,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:48,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49116 deadline: 1730972328647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:48,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:48,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:48,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:48,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:48,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:48,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49162 deadline: 1730972328655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:48,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49086 deadline: 1730972328648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:48,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:48,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:48,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:48,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:48,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49148 deadline: 1730972328657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:48,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:48,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49098 deadline: 1730972328658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:48,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:48,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:48,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:48,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:48,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:48,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:48,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:48,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:48,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:48,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-07T09:37:48,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:48,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49116 deadline: 1730972328773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:48,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:48,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:48,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:48,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:48,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49086 deadline: 1730972328773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:48,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:48,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:48,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:48,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:48,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49162 deadline: 1730972328783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:48,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:48,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49148 deadline: 1730972328784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:48,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:48,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:48,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:48,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:48,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:48,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:48,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:48,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49098 deadline: 1730972328784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:48,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:48,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:48,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:48,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:48,999 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107e9b0c1c8a0614ffb876665f7971e996c_7f7127bc07c9cd4d3f409040fbdd89d3 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107e9b0c1c8a0614ffb876665f7971e996c_7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:49,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:49,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49116 deadline: 1730972328993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:49,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:49,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49086 deadline: 1730972328993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:49,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:49,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:49,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:49,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/d24b2c29e349497e92907e822e618840, store: [table=TestAcidGuarantees family=A region=7f7127bc07c9cd4d3f409040fbdd89d3] 2024-11-07T09:37:49,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:49,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:49,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:49,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/d24b2c29e349497e92907e822e618840 is 175, key is test_row_0/A:col10/1730972267910/Put/seqid=0 2024-11-07T09:37:49,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742248_1424 (size=31105) 2024-11-07T09:37:49,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-07T09:37:49,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:49,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49162 deadline: 1730972329002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:49,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:49,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49148 deadline: 1730972329002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:49,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:49,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:49,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:49,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:49,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49098 deadline: 1730972329003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:49,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:49,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:49,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:49,017 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:49,017 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:49,017 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:49,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:49,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49116 deadline: 1730972329303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:49,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:49,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:49,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:49,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:49,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49086 deadline: 1730972329313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:49,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:49,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:49,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:49,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:49,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49148 deadline: 1730972329322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:49,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:49,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:49,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:49,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49098 deadline: 1730972329323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:49,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:49,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:49,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49162 deadline: 1730972329323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:49,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:49,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:49,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:49,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:49,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:49,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:49,407 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/d24b2c29e349497e92907e822e618840 2024-11-07T09:37:49,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/5945d159fb89457ba2914866c9686670 is 50, key is test_row_0/B:col10/1730972267910/Put/seqid=0 2024-11-07T09:37:49,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742249_1425 (size=12151) 2024-11-07T09:37:49,439 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/5945d159fb89457ba2914866c9686670 2024-11-07T09:37:49,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/f99604ef6fcd40ef8ffe395ea540a664 is 50, key is test_row_0/C:col10/1730972267910/Put/seqid=0 2024-11-07T09:37:49,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742250_1426 (size=12151) 2024-11-07T09:37:49,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-07T09:37:49,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:49,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49116 deadline: 1730972329823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:49,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:49,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49086 deadline: 1730972329823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:49,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:49,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:49,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:49,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:49,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:49,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:49,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:49,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49148 deadline: 1730972329843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:49,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:49,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49162 deadline: 1730972329843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:49,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:49,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:49,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49098 deadline: 1730972329844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:49,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:49,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:49,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:49,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:49,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:49,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:49,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:49,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:49,904 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/f99604ef6fcd40ef8ffe395ea540a664 2024-11-07T09:37:49,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/d24b2c29e349497e92907e822e618840 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/d24b2c29e349497e92907e822e618840 2024-11-07T09:37:49,920 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/d24b2c29e349497e92907e822e618840, entries=150, sequenceid=170, filesize=30.4 K 2024-11-07T09:37:49,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/5945d159fb89457ba2914866c9686670 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/5945d159fb89457ba2914866c9686670 2024-11-07T09:37:49,925 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/5945d159fb89457ba2914866c9686670, entries=150, sequenceid=170, filesize=11.9 K 2024-11-07T09:37:49,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/f99604ef6fcd40ef8ffe395ea540a664 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/f99604ef6fcd40ef8ffe395ea540a664 2024-11-07T09:37:49,929 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/f99604ef6fcd40ef8ffe395ea540a664, entries=150, sequenceid=170, filesize=11.9 K 2024-11-07T09:37:49,929 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 7f7127bc07c9cd4d3f409040fbdd89d3 in 1387ms, sequenceid=170, compaction requested=true 2024-11-07T09:37:49,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 7f7127bc07c9cd4d3f409040fbdd89d3: 2024-11-07T09:37:49,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:49,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-07T09:37:49,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-07T09:37:49,933 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-07T09:37:49,933 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5420 sec 2024-11-07T09:37:49,935 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 1.5480 sec 2024-11-07T09:37:50,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-07T09:37:50,534 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:37:50,535 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T09:37:50,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-07T09:37:50,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-07T09:37:50,538 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T09:37:50,545 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T09:37:50,547 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T09:37:50,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-07T09:37:50,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-07T09:37:50,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:50,701 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 7f7127bc07c9cd4d3f409040fbdd89d3 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T09:37:50,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107def42aa69ed4417bb5c45b698b330ad3_7f7127bc07c9cd4d3f409040fbdd89d3 is 50, key is test_row_0/A:col10/1730972268653/Put/seqid=0 2024-11-07T09:37:50,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742251_1427 (size=12304) 2024-11-07T09:37:50,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:50,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. as already flushing 2024-11-07T09:37:50,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-07T09:37:50,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:50,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49098 deadline: 1730972330873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:50,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:50,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49148 deadline: 1730972330873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:50,885 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 9f1eb259a29b,42895,1730972153970 for 'test_row_0' in 7f7127bc07c9cd4d3f409040fbdd89d3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T09:37:50,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:50,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:50,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:50,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49116 deadline: 1730972330876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:50,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:50,885 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 9f1eb259a29b,42895,1730972153970 for 'test_row_1' in 7f7127bc07c9cd4d3f409040fbdd89d3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T09:37:50,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:50,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:50,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:50,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:50,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:50,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:50,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:50,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49086 deadline: 1730972330882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:50,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:50,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:50,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:50,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:50,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49162 deadline: 1730972330884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:50,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:50,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:50,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:51,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:51,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49116 deadline: 1730972330993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:51,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:51,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:51,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49086 deadline: 1730972330993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:51,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:51,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:51,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:51,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:51,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:51,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:51,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49162 deadline: 1730972331005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:51,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:51,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:51,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:51,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:51,122 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107def42aa69ed4417bb5c45b698b330ad3_7f7127bc07c9cd4d3f409040fbdd89d3 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107def42aa69ed4417bb5c45b698b330ad3_7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:51,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/3606f7d70d0f4509aded546ad685a6b3, store: [table=TestAcidGuarantees family=A region=7f7127bc07c9cd4d3f409040fbdd89d3] 2024-11-07T09:37:51,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/3606f7d70d0f4509aded546ad685a6b3 is 175, key is test_row_0/A:col10/1730972268653/Put/seqid=0 2024-11-07T09:37:51,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742252_1428 (size=31105) 2024-11-07T09:37:51,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-07T09:37:51,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:51,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49116 deadline: 1730972331213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:51,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:51,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49086 deadline: 1730972331214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:51,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:51,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:51,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:51,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:51,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:51,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:51,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:51,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49162 deadline: 1730972331223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:51,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:51,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:51,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:51,531 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=195, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/3606f7d70d0f4509aded546ad685a6b3 2024-11-07T09:37:51,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:51,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49116 deadline: 1730972331533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:51,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:51,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49086 deadline: 1730972331533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:51,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:51,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:51,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:51,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:51,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:51,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:51,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/efe526cc29d04dc783d9ec939a1d97cc is 50, key is test_row_0/B:col10/1730972268653/Put/seqid=0 2024-11-07T09:37:51,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742253_1429 (size=12151) 2024-11-07T09:37:51,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:51,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49162 deadline: 1730972331542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:51,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:51,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:51,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:51,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-07T09:37:51,948 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/efe526cc29d04dc783d9ec939a1d97cc 2024-11-07T09:37:51,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/dc3b6d8dcead47c9a1bf8139febe8d8a is 50, key is test_row_0/C:col10/1730972268653/Put/seqid=0 2024-11-07T09:37:51,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742254_1430 (size=12151) 2024-11-07T09:37:52,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:52,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49116 deadline: 1730972332043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:52,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:52,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:52,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:52,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49086 deadline: 1730972332043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:52,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:52,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:52,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:52,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:52,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:52,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49162 deadline: 1730972332063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:52,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:52,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:52,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:52,364 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/dc3b6d8dcead47c9a1bf8139febe8d8a 2024-11-07T09:37:52,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/3606f7d70d0f4509aded546ad685a6b3 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/3606f7d70d0f4509aded546ad685a6b3 2024-11-07T09:37:52,372 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/3606f7d70d0f4509aded546ad685a6b3, entries=150, sequenceid=195, filesize=30.4 K 2024-11-07T09:37:52,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/efe526cc29d04dc783d9ec939a1d97cc as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/efe526cc29d04dc783d9ec939a1d97cc 2024-11-07T09:37:52,377 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/efe526cc29d04dc783d9ec939a1d97cc, entries=150, sequenceid=195, filesize=11.9 K 2024-11-07T09:37:52,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/dc3b6d8dcead47c9a1bf8139febe8d8a as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/dc3b6d8dcead47c9a1bf8139febe8d8a 2024-11-07T09:37:52,382 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/dc3b6d8dcead47c9a1bf8139febe8d8a, entries=150, sequenceid=195, filesize=11.9 K 2024-11-07T09:37:52,383 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 7f7127bc07c9cd4d3f409040fbdd89d3 in 1682ms, sequenceid=195, compaction requested=true 2024-11-07T09:37:52,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 7f7127bc07c9cd4d3f409040fbdd89d3: 2024-11-07T09:37:52,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:52,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-07T09:37:52,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-07T09:37:52,388 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-07T09:37:52,388 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8380 sec 2024-11-07T09:37:52,392 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.8550 sec 2024-11-07T09:37:52,442 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T09:37:52,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-07T09:37:52,693 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:37:52,694 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T09:37:52,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-07T09:37:52,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-07T09:37:52,697 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T09:37:52,698 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T09:37:52,698 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T09:37:52,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-07T09:37:52,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-07T09:37:52,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:52,856 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 7f7127bc07c9cd4d3f409040fbdd89d3 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T09:37:52,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107d02e46862f954245a0acd69dbfca48c7_7f7127bc07c9cd4d3f409040fbdd89d3 is 50, key is test_row_0/A:col10/1730972270880/Put/seqid=0 2024-11-07T09:37:52,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742255_1431 (size=12304) 2024-11-07T09:37:52,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:52,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. as already flushing 2024-11-07T09:37:53,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-07T09:37:53,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:53,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49098 deadline: 1730972333045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:53,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:53,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:53,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:53,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:53,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49148 deadline: 1730972333046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:53,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:53,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:53,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:53,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:53,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49116 deadline: 1730972333063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:53,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:53,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49086 deadline: 1730972333063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:53,077 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 9f1eb259a29b,42895,1730972153970 for 'test_row_0' in 7f7127bc07c9cd4d3f409040fbdd89d3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T09:37:53,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:53,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:53,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:53,077 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 9f1eb259a29b,42895,1730972153970 for 'test_row_0' in 7f7127bc07c9cd4d3f409040fbdd89d3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T09:37:53,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:53,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:53,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:53,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:53,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49162 deadline: 1730972333092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:53,097 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 9f1eb259a29b,42895,1730972153970 for 'test_row_1' in 7f7127bc07c9cd4d3f409040fbdd89d3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T09:37:53,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:53,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:53,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:53,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:53,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49098 deadline: 1730972333163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:53,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:53,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:53,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:53,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:53,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49148 deadline: 1730972333172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:53,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:53,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:53,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:53,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:53,291 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107d02e46862f954245a0acd69dbfca48c7_7f7127bc07c9cd4d3f409040fbdd89d3 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107d02e46862f954245a0acd69dbfca48c7_7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:53,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/f9790cc940e34a1688a368e023ac45a3, store: [table=TestAcidGuarantees family=A region=7f7127bc07c9cd4d3f409040fbdd89d3] 2024-11-07T09:37:53,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/f9790cc940e34a1688a368e023ac45a3 is 175, key is test_row_0/A:col10/1730972270880/Put/seqid=0 2024-11-07T09:37:53,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742256_1432 (size=31105) 2024-11-07T09:37:53,302 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=206, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/f9790cc940e34a1688a368e023ac45a3 2024-11-07T09:37:53,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-07T09:37:53,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/4a67d96c8b354b7b86dd8daec2d2bb70 is 50, key is test_row_0/B:col10/1730972270880/Put/seqid=0 2024-11-07T09:37:53,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742257_1433 (size=12151) 2024-11-07T09:37:53,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:53,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49098 deadline: 1730972333373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:53,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:53,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:53,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:53,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:53,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49148 deadline: 1730972333393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:53,402 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:53,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:53,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:53,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:53,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49098 deadline: 1730972333693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:53,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:53,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:53,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:53,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:53,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49148 deadline: 1730972333713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:53,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:53,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:53,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:53,745 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/4a67d96c8b354b7b86dd8daec2d2bb70 2024-11-07T09:37:53,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/95fbe683c9894212984e4768d056a8e0 is 50, key is test_row_0/C:col10/1730972270880/Put/seqid=0 2024-11-07T09:37:53,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742258_1434 (size=12151) 2024-11-07T09:37:53,757 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/95fbe683c9894212984e4768d056a8e0 2024-11-07T09:37:53,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/f9790cc940e34a1688a368e023ac45a3 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/f9790cc940e34a1688a368e023ac45a3 2024-11-07T09:37:53,768 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/f9790cc940e34a1688a368e023ac45a3, entries=150, sequenceid=206, filesize=30.4 K 2024-11-07T09:37:53,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/4a67d96c8b354b7b86dd8daec2d2bb70 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/4a67d96c8b354b7b86dd8daec2d2bb70 2024-11-07T09:37:53,777 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/4a67d96c8b354b7b86dd8daec2d2bb70, entries=150, sequenceid=206, filesize=11.9 K 2024-11-07T09:37:53,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/95fbe683c9894212984e4768d056a8e0 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/95fbe683c9894212984e4768d056a8e0 2024-11-07T09:37:53,781 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/95fbe683c9894212984e4768d056a8e0, entries=150, sequenceid=206, filesize=11.9 K 2024-11-07T09:37:53,783 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 7f7127bc07c9cd4d3f409040fbdd89d3 in 927ms, sequenceid=206, compaction requested=true 2024-11-07T09:37:53,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 7f7127bc07c9cd4d3f409040fbdd89d3: 2024-11-07T09:37:53,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:53,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-07T09:37:53,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-07T09:37:53,787 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-07T09:37:53,787 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0870 sec 2024-11-07T09:37:53,789 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.0940 sec 2024-11-07T09:37:53,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-07T09:37:53,833 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:37:53,834 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T09:37:53,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-07T09:37:53,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-07T09:37:53,836 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T09:37:53,838 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T09:37:53,838 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T09:37:53,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-07T09:37:53,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-07T09:37:53,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:53,992 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 7f7127bc07c9cd4d3f409040fbdd89d3 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T09:37:54,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107161dd5994eb7430bbfc03e2952511750_7f7127bc07c9cd4d3f409040fbdd89d3 is 50, key is test_row_0/A:col10/1730972273044/Put/seqid=0 2024-11-07T09:37:54,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742259_1435 (size=12304) 2024-11-07T09:37:54,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:54,018 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107161dd5994eb7430bbfc03e2952511750_7f7127bc07c9cd4d3f409040fbdd89d3 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107161dd5994eb7430bbfc03e2952511750_7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:54,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/5af75993776648d3819f1a6b869e7b46, store: [table=TestAcidGuarantees family=A region=7f7127bc07c9cd4d3f409040fbdd89d3] 2024-11-07T09:37:54,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/5af75993776648d3819f1a6b869e7b46 is 175, key is test_row_0/A:col10/1730972273044/Put/seqid=0 2024-11-07T09:37:54,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742260_1436 (size=31105) 2024-11-07T09:37:54,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-07T09:37:54,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:54,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. as already flushing 2024-11-07T09:37:54,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:54,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49148 deadline: 1730972334263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:54,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:54,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49098 deadline: 1730972334270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:54,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:54,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:54,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:54,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:54,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:54,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:54,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:54,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49148 deadline: 1730972334383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:54,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:54,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:54,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:54,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:54,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49098 deadline: 1730972334383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:54,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:54,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:54,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:54,433 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=231, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/5af75993776648d3819f1a6b869e7b46 2024-11-07T09:37:54,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/e388c8afb80443f384b9d883cd5fc67d is 50, key is test_row_0/B:col10/1730972273044/Put/seqid=0 2024-11-07T09:37:54,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742261_1437 (size=12151) 2024-11-07T09:37:54,455 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/e388c8afb80443f384b9d883cd5fc67d 2024-11-07T09:37:54,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/13a68634aa374946abe37fb2176bfaae is 50, key is test_row_0/C:col10/1730972273044/Put/seqid=0 2024-11-07T09:37:54,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-07T09:37:54,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742262_1438 (size=12151) 2024-11-07T09:37:54,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:54,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49098 deadline: 1730972334608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:54,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:54,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:54,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:54,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:54,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49148 deadline: 1730972334608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:54,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:54,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:54,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:54,870 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/13a68634aa374946abe37fb2176bfaae 2024-11-07T09:37:54,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/5af75993776648d3819f1a6b869e7b46 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/5af75993776648d3819f1a6b869e7b46 2024-11-07T09:37:54,880 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/5af75993776648d3819f1a6b869e7b46, entries=150, sequenceid=231, filesize=30.4 K 2024-11-07T09:37:54,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/e388c8afb80443f384b9d883cd5fc67d as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/e388c8afb80443f384b9d883cd5fc67d 2024-11-07T09:37:54,884 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/e388c8afb80443f384b9d883cd5fc67d, entries=150, sequenceid=231, filesize=11.9 K 2024-11-07T09:37:54,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/13a68634aa374946abe37fb2176bfaae as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/13a68634aa374946abe37fb2176bfaae 2024-11-07T09:37:54,888 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/13a68634aa374946abe37fb2176bfaae, entries=150, sequenceid=231, filesize=11.9 K 2024-11-07T09:37:54,889 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 7f7127bc07c9cd4d3f409040fbdd89d3 in 898ms, sequenceid=231, compaction requested=true 2024-11-07T09:37:54,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 7f7127bc07c9cd4d3f409040fbdd89d3: 2024-11-07T09:37:54,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:54,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-07T09:37:54,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-07T09:37:54,893 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-07T09:37:54,893 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0530 sec 2024-11-07T09:37:54,895 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.0600 sec 2024-11-07T09:37:54,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:54,933 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:37:54,933 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7f7127bc07c9cd4d3f409040fbdd89d3 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-07T09:37:54,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411070e232ac73d7b42c3855d0cf89c3ed2fa_7f7127bc07c9cd4d3f409040fbdd89d3 is 50, key is test_row_0/A:col10/1730972274931/Put/seqid=0 2024-11-07T09:37:54,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742263_1439 (size=14794) 2024-11-07T09:37:54,959 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:54,964 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411070e232ac73d7b42c3855d0cf89c3ed2fa_7f7127bc07c9cd4d3f409040fbdd89d3 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070e232ac73d7b42c3855d0cf89c3ed2fa_7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:54,967 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/42d843a9bfd14132afdf73d10a68750e, store: [table=TestAcidGuarantees family=A region=7f7127bc07c9cd4d3f409040fbdd89d3] 2024-11-07T09:37:54,968 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/42d843a9bfd14132afdf73d10a68750e is 175, key is test_row_0/A:col10/1730972274931/Put/seqid=0 2024-11-07T09:37:54,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-07T09:37:54,973 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:37:54,974 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T09:37:54,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-07T09:37:54,976 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T09:37:54,977 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T09:37:54,977 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T09:37:54,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-07T09:37:54,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742264_1440 (size=39749) 2024-11-07T09:37:55,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-07T09:37:55,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:55,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49116 deadline: 1730972335102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:55,118 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 9f1eb259a29b,42895,1730972153970 for 'test_row_0' in 7f7127bc07c9cd4d3f409040fbdd89d3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4242 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T09:37:55,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:55,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:55,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:55,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:55,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49148 deadline: 1730972335110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:55,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:55,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:55,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:55,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49098 deadline: 1730972335111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:55,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:55,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:55,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:55,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:55,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49086 deadline: 1730972335112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:55,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:55,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:55,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:55,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:55,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:55,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49162 deadline: 1730972335113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:55,128 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 9f1eb259a29b,42895,1730972153970 for 'test_row_1' in 7f7127bc07c9cd4d3f409040fbdd89d3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4244 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T09:37:55,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:55,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:55,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:55,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-07T09:37:55,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:55,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. as already flushing 2024-11-07T09:37:55,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:55,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:55,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:55,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:55,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:55,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49148 deadline: 1730972335233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:55,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:55,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:55,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49086 deadline: 1730972335233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:55,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49098 deadline: 1730972335233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:55,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:55,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:55,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:55,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:55,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:55,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:55,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:55,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:55,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:55,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-07T09:37:55,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:55,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. as already flushing 2024-11-07T09:37:55,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:55,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:55,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:55,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:55,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-07T09:37:55,384 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=243, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/42d843a9bfd14132afdf73d10a68750e 2024-11-07T09:37:55,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/59c29f8629c64d6598eced49bd9d06dd is 50, key is test_row_0/B:col10/1730972274931/Put/seqid=0 2024-11-07T09:37:55,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742265_1441 (size=12151) 2024-11-07T09:37:55,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-07T09:37:55,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:55,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. as already flushing 2024-11-07T09:37:55,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:55,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:55,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:55,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:55,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:55,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49148 deadline: 1730972335453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:55,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:55,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:55,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:55,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:55,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49086 deadline: 1730972335453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:55,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:55,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:55,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:55,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:55,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49098 deadline: 1730972335453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:55,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:55,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:55,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:55,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-07T09:37:55,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:55,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. as already flushing 2024-11-07T09:37:55,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:55,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:55,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:55,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:55,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-07T09:37:55,743 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-07T09:37:55,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:55,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. as already flushing 2024-11-07T09:37:55,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:55,743 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:55,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:55,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:55,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:55,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49086 deadline: 1730972335773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:55,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:55,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:55,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49148 deadline: 1730972335774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:55,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:55,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:55,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:55,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49098 deadline: 1730972335774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:55,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:55,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:55,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:55,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:55,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:55,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:55,803 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/59c29f8629c64d6598eced49bd9d06dd 2024-11-07T09:37:55,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/b6a90f0f349d47a2984663e3b3967f4e is 50, key is test_row_0/C:col10/1730972274931/Put/seqid=0 2024-11-07T09:37:55,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742266_1442 (size=12151) 2024-11-07T09:37:55,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-07T09:37:55,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:55,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. as already flushing 2024-11-07T09:37:55,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:55,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:55,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:55,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:56,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-07T09:37:56,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:56,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. as already flushing 2024-11-07T09:37:56,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:56,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:56,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:56,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:56,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-07T09:37:56,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-07T09:37:56,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:56,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. as already flushing 2024-11-07T09:37:56,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:56,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:56,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:56,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:56,220 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/b6a90f0f349d47a2984663e3b3967f4e 2024-11-07T09:37:56,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/42d843a9bfd14132afdf73d10a68750e as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/42d843a9bfd14132afdf73d10a68750e 2024-11-07T09:37:56,230 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/42d843a9bfd14132afdf73d10a68750e, entries=200, sequenceid=243, filesize=38.8 K 2024-11-07T09:37:56,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/59c29f8629c64d6598eced49bd9d06dd as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/59c29f8629c64d6598eced49bd9d06dd 2024-11-07T09:37:56,234 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/59c29f8629c64d6598eced49bd9d06dd, entries=150, sequenceid=243, filesize=11.9 K 2024-11-07T09:37:56,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/b6a90f0f349d47a2984663e3b3967f4e as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/b6a90f0f349d47a2984663e3b3967f4e 2024-11-07T09:37:56,239 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/b6a90f0f349d47a2984663e3b3967f4e, entries=150, sequenceid=243, filesize=11.9 K 2024-11-07T09:37:56,240 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 7f7127bc07c9cd4d3f409040fbdd89d3 in 1307ms, sequenceid=243, compaction requested=true 2024-11-07T09:37:56,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7f7127bc07c9cd4d3f409040fbdd89d3: 2024-11-07T09:37:56,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f7127bc07c9cd4d3f409040fbdd89d3:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T09:37:56,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:56,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f7127bc07c9cd4d3f409040fbdd89d3:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T09:37:56,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:56,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f7127bc07c9cd4d3f409040fbdd89d3:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:37:56,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:37:56,240 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-07T09:37:56,240 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-07T09:37:56,242 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 235281 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-07T09:37:56,242 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 85315 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-07T09:37:56,242 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 7f7127bc07c9cd4d3f409040fbdd89d3/B is initiating minor compaction (all files) 2024-11-07T09:37:56,242 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 7f7127bc07c9cd4d3f409040fbdd89d3/A is initiating minor compaction (all files) 2024-11-07T09:37:56,242 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7f7127bc07c9cd4d3f409040fbdd89d3/B in TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:56,242 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7f7127bc07c9cd4d3f409040fbdd89d3/A in TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:56,242 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/e85fe8674a744497b88d0dfa62f96685, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/e0482d64ea8b462ea5262df9edee7153, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/d24b2c29e349497e92907e822e618840, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/3606f7d70d0f4509aded546ad685a6b3, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/f9790cc940e34a1688a368e023ac45a3, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/5af75993776648d3819f1a6b869e7b46, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/42d843a9bfd14132afdf73d10a68750e] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp, totalSize=229.8 K 2024-11-07T09:37:56,242 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/2e42873a33c84082ad7079c2d11b646d, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/ad0eed3e07204969a202054406042afc, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/5945d159fb89457ba2914866c9686670, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/efe526cc29d04dc783d9ec939a1d97cc, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/4a67d96c8b354b7b86dd8daec2d2bb70, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/e388c8afb80443f384b9d883cd5fc67d, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/59c29f8629c64d6598eced49bd9d06dd] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp, totalSize=83.3 K 2024-11-07T09:37:56,242 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=9 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:56,242 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. files: [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/e85fe8674a744497b88d0dfa62f96685, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/e0482d64ea8b462ea5262df9edee7153, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/d24b2c29e349497e92907e822e618840, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/3606f7d70d0f4509aded546ad685a6b3, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/f9790cc940e34a1688a368e023ac45a3, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/5af75993776648d3819f1a6b869e7b46, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/42d843a9bfd14132afdf73d10a68750e] 2024-11-07T09:37:56,242 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting e85fe8674a744497b88d0dfa62f96685, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1730972266643 2024-11-07T09:37:56,242 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e42873a33c84082ad7079c2d11b646d, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1730972266643 2024-11-07T09:37:56,243 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0482d64ea8b462ea5262df9edee7153, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1730972266699 2024-11-07T09:37:56,244 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting ad0eed3e07204969a202054406042afc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1730972266705 2024-11-07T09:37:56,244 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting d24b2c29e349497e92907e822e618840, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1730972267909 2024-11-07T09:37:56,244 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 5945d159fb89457ba2914866c9686670, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1730972267909 2024-11-07T09:37:56,244 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3606f7d70d0f4509aded546ad685a6b3, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1730972268646 2024-11-07T09:37:56,244 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting efe526cc29d04dc783d9ec939a1d97cc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1730972268646 2024-11-07T09:37:56,244 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9790cc940e34a1688a368e023ac45a3, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1730972270869 2024-11-07T09:37:56,244 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a67d96c8b354b7b86dd8daec2d2bb70, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1730972270869 2024-11-07T09:37:56,245 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5af75993776648d3819f1a6b869e7b46, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1730972273023 2024-11-07T09:37:56,245 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting e388c8afb80443f384b9d883cd5fc67d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1730972273023 2024-11-07T09:37:56,245 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42d843a9bfd14132afdf73d10a68750e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1730972274255 2024-11-07T09:37:56,245 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 59c29f8629c64d6598eced49bd9d06dd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1730972274261 2024-11-07T09:37:56,263 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7f7127bc07c9cd4d3f409040fbdd89d3#B#compaction#377 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:37:56,264 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/98ecf4942b1d45a2954168f26dd43cfd is 50, key is test_row_0/B:col10/1730972274931/Put/seqid=0 2024-11-07T09:37:56,266 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7f7127bc07c9cd4d3f409040fbdd89d3] 2024-11-07T09:37:56,281 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411075794d6cdcac84c7faefdc1835dd44be4_7f7127bc07c9cd4d3f409040fbdd89d3 store=[table=TestAcidGuarantees family=A region=7f7127bc07c9cd4d3f409040fbdd89d3] 2024-11-07T09:37:56,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742267_1443 (size=12697) 2024-11-07T09:37:56,286 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411075794d6cdcac84c7faefdc1835dd44be4_7f7127bc07c9cd4d3f409040fbdd89d3, store=[table=TestAcidGuarantees family=A region=7f7127bc07c9cd4d3f409040fbdd89d3] 2024-11-07T09:37:56,286 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411075794d6cdcac84c7faefdc1835dd44be4_7f7127bc07c9cd4d3f409040fbdd89d3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7f7127bc07c9cd4d3f409040fbdd89d3] 2024-11-07T09:37:56,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:56,290 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:37:56,290 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7f7127bc07c9cd4d3f409040fbdd89d3 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T09:37:56,295 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/98ecf4942b1d45a2954168f26dd43cfd as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/98ecf4942b1d45a2954168f26dd43cfd 2024-11-07T09:37:56,301 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in 7f7127bc07c9cd4d3f409040fbdd89d3/B of 7f7127bc07c9cd4d3f409040fbdd89d3 into 98ecf4942b1d45a2954168f26dd43cfd(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:37:56,301 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7f7127bc07c9cd4d3f409040fbdd89d3: 2024-11-07T09:37:56,301 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., storeName=7f7127bc07c9cd4d3f409040fbdd89d3/B, priority=9, startTime=1730972276240; duration=0sec 2024-11-07T09:37:56,301 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:37:56,301 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f7127bc07c9cd4d3f409040fbdd89d3:B 2024-11-07T09:37:56,302 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-07T09:37:56,303 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 85315 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-07T09:37:56,303 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 7f7127bc07c9cd4d3f409040fbdd89d3/C is initiating minor compaction (all files) 2024-11-07T09:37:56,303 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7f7127bc07c9cd4d3f409040fbdd89d3/C in TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:56,304 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/34d4c3063d134d319370c82f642ebfe9, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/37c7bfcefea64727bad363f9c267f95a, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/f99604ef6fcd40ef8ffe395ea540a664, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/dc3b6d8dcead47c9a1bf8139febe8d8a, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/95fbe683c9894212984e4768d056a8e0, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/13a68634aa374946abe37fb2176bfaae, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/b6a90f0f349d47a2984663e3b3967f4e] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp, totalSize=83.3 K 2024-11-07T09:37:56,304 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 34d4c3063d134d319370c82f642ebfe9, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1730972266643 2024-11-07T09:37:56,304 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 37c7bfcefea64727bad363f9c267f95a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1730972266705 2024-11-07T09:37:56,305 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting f99604ef6fcd40ef8ffe395ea540a664, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1730972267909 2024-11-07T09:37:56,305 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting dc3b6d8dcead47c9a1bf8139febe8d8a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1730972268646 2024-11-07T09:37:56,305 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 95fbe683c9894212984e4768d056a8e0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1730972270869 2024-11-07T09:37:56,305 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 13a68634aa374946abe37fb2176bfaae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1730972273023 2024-11-07T09:37:56,306 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting b6a90f0f349d47a2984663e3b3967f4e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1730972274261 2024-11-07T09:37:56,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411076460c27e4f5345f8a4cb01e5a59ef867_7f7127bc07c9cd4d3f409040fbdd89d3 is 50, key is test_row_0/A:col10/1730972275111/Put/seqid=0 2024-11-07T09:37:56,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742268_1444 (size=4469) 2024-11-07T09:37:56,323 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7f7127bc07c9cd4d3f409040fbdd89d3#C#compaction#380 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T09:37:56,323 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/d8ae26052b2d47cfbc04cd56a219ba9b is 50, key is test_row_0/C:col10/1730972274931/Put/seqid=0 2024-11-07T09:37:56,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742269_1445 (size=14994) 2024-11-07T09:37:56,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742270_1446 (size=12697) 2024-11-07T09:37:56,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:56,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49148 deadline: 1730972336332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:56,350 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:56,350 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:56,350 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:56,350 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/d8ae26052b2d47cfbc04cd56a219ba9b as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/d8ae26052b2d47cfbc04cd56a219ba9b 2024-11-07T09:37:56,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:56,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49098 deadline: 1730972336334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:56,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:56,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:56,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:56,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:56,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49086 deadline: 1730972336336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:56,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-07T09:37:56,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:56,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. as already flushing 2024-11-07T09:37:56,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:56,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:56,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:56,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:56,359 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in 7f7127bc07c9cd4d3f409040fbdd89d3/C of 7f7127bc07c9cd4d3f409040fbdd89d3 into d8ae26052b2d47cfbc04cd56a219ba9b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:37:56,359 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7f7127bc07c9cd4d3f409040fbdd89d3: 2024-11-07T09:37:56,359 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., storeName=7f7127bc07c9cd4d3f409040fbdd89d3/C, priority=9, startTime=1730972276240; duration=0sec 2024-11-07T09:37:56,359 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:56,359 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f7127bc07c9cd4d3f409040fbdd89d3:C 2024-11-07T09:37:56,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:56,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49148 deadline: 1730972336453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:56,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:56,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:56,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:56,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:56,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49098 deadline: 1730972336463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:56,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:56,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:56,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:56,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:56,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49086 deadline: 1730972336463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:56,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:56,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:56,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:56,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-07T09:37:56,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:56,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. as already flushing 2024-11-07T09:37:56,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:56,510 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:56,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:56,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:56,631 DEBUG [regionserver/9f1eb259a29b:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.3076923076923077, tune throughput to 65.38 MB/second 2024-11-07T09:37:56,633 WARN [regionserver/9f1eb259a29b:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-11-07T09:37:56,633 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9f1eb259a29b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/0d4b997726f14c8396e95dfa0ff94968, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/4b041091f08043198cb88ed817b01bde, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/ebb241d7d3ad46aaac1993829f6e77f1, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/479057baa25541a19a663c98d1456544, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/f370f044157f4e4bb8a454445644665f, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/4f4c2857859c4d98beb897acca385438, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/e45294a9e04d4af2a8fc5c1cf47d781e, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/2d2b4eed4ca147e4a30c73ece6202340, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/c53bee323e764e9cad543e57e12e7389] to archive 2024-11-07T09:37:56,634 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9f1eb259a29b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T09:37:56,635 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9f1eb259a29b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/0d4b997726f14c8396e95dfa0ff94968 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/0d4b997726f14c8396e95dfa0ff94968 2024-11-07T09:37:56,636 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9f1eb259a29b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/4b041091f08043198cb88ed817b01bde to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/4b041091f08043198cb88ed817b01bde 2024-11-07T09:37:56,638 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9f1eb259a29b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/ebb241d7d3ad46aaac1993829f6e77f1 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/ebb241d7d3ad46aaac1993829f6e77f1 2024-11-07T09:37:56,639 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9f1eb259a29b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/479057baa25541a19a663c98d1456544 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/479057baa25541a19a663c98d1456544 2024-11-07T09:37:56,641 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9f1eb259a29b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/f370f044157f4e4bb8a454445644665f to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/f370f044157f4e4bb8a454445644665f 2024-11-07T09:37:56,642 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9f1eb259a29b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/4f4c2857859c4d98beb897acca385438 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/4f4c2857859c4d98beb897acca385438 2024-11-07T09:37:56,643 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9f1eb259a29b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/e45294a9e04d4af2a8fc5c1cf47d781e to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/e45294a9e04d4af2a8fc5c1cf47d781e 2024-11-07T09:37:56,644 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9f1eb259a29b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/2d2b4eed4ca147e4a30c73ece6202340 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/2d2b4eed4ca147e4a30c73ece6202340 2024-11-07T09:37:56,645 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9f1eb259a29b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/c53bee323e764e9cad543e57e12e7389 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/c53bee323e764e9cad543e57e12e7389 2024-11-07T09:37:56,647 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9f1eb259a29b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/effc86c3e9914434b79bc59a0873480d, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/8177ad162a8947078d529b00499524f7, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/77381fcde3234fbdae705254816a71ed, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/d8c9a79ac6f14c8abdfffdcc48e6993e, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/35c0cd9cb5b245fdb779b76abddfc436, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/a6fbff66161c4796a4c167a555da83c9, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/76f51d0333b14f1c9d929b2f58befe22, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/21a8216b215449d3aa9dd04cbb39a6ba, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/2e42873a33c84082ad7079c2d11b646d, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/c799eec1b6af4085bcfa558f44f51e77, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/ad0eed3e07204969a202054406042afc, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/5945d159fb89457ba2914866c9686670, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/efe526cc29d04dc783d9ec939a1d97cc, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/4a67d96c8b354b7b86dd8daec2d2bb70, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/e388c8afb80443f384b9d883cd5fc67d, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/59c29f8629c64d6598eced49bd9d06dd] to archive 2024-11-07T09:37:56,648 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9f1eb259a29b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T09:37:56,649 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9f1eb259a29b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/effc86c3e9914434b79bc59a0873480d to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/effc86c3e9914434b79bc59a0873480d 2024-11-07T09:37:56,651 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9f1eb259a29b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/8177ad162a8947078d529b00499524f7 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/8177ad162a8947078d529b00499524f7 2024-11-07T09:37:56,652 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9f1eb259a29b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/77381fcde3234fbdae705254816a71ed to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/77381fcde3234fbdae705254816a71ed 2024-11-07T09:37:56,653 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9f1eb259a29b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/d8c9a79ac6f14c8abdfffdcc48e6993e to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/d8c9a79ac6f14c8abdfffdcc48e6993e 2024-11-07T09:37:56,656 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9f1eb259a29b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/35c0cd9cb5b245fdb779b76abddfc436 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/35c0cd9cb5b245fdb779b76abddfc436 2024-11-07T09:37:56,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-07T09:37:56,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:56,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. as already flushing 2024-11-07T09:37:56,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:56,664 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:56,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:56,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:56,665 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9f1eb259a29b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/a6fbff66161c4796a4c167a555da83c9 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/a6fbff66161c4796a4c167a555da83c9 2024-11-07T09:37:56,667 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9f1eb259a29b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/76f51d0333b14f1c9d929b2f58befe22 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/76f51d0333b14f1c9d929b2f58befe22 2024-11-07T09:37:56,668 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9f1eb259a29b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/21a8216b215449d3aa9dd04cbb39a6ba to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/21a8216b215449d3aa9dd04cbb39a6ba 2024-11-07T09:37:56,670 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9f1eb259a29b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/2e42873a33c84082ad7079c2d11b646d to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/2e42873a33c84082ad7079c2d11b646d 2024-11-07T09:37:56,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:56,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49148 deadline: 1730972336663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:56,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:56,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:56,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:56,671 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9f1eb259a29b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/c799eec1b6af4085bcfa558f44f51e77 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/c799eec1b6af4085bcfa558f44f51e77 2024-11-07T09:37:56,673 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9f1eb259a29b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/ad0eed3e07204969a202054406042afc to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/ad0eed3e07204969a202054406042afc 2024-11-07T09:37:56,674 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9f1eb259a29b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/5945d159fb89457ba2914866c9686670 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/5945d159fb89457ba2914866c9686670 2024-11-07T09:37:56,675 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9f1eb259a29b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/efe526cc29d04dc783d9ec939a1d97cc to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/efe526cc29d04dc783d9ec939a1d97cc 2024-11-07T09:37:56,676 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9f1eb259a29b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/4a67d96c8b354b7b86dd8daec2d2bb70 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/4a67d96c8b354b7b86dd8daec2d2bb70 2024-11-07T09:37:56,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:56,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49098 deadline: 1730972336683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:56,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:56,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49086 deadline: 1730972336683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:56,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:56,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:56,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:56,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:56,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:56,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:56,692 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9f1eb259a29b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/e388c8afb80443f384b9d883cd5fc67d to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/e388c8afb80443f384b9d883cd5fc67d 2024-11-07T09:37:56,694 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9f1eb259a29b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/59c29f8629c64d6598eced49bd9d06dd to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/59c29f8629c64d6598eced49bd9d06dd 2024-11-07T09:37:56,697 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9f1eb259a29b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/c5d0b0e8a858466a8232572debb86dfd, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/bb491638daf2457b8c35e2b3822daea2, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/88bd9057ff244159abee084c8557ee1b, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/efa6f537a1ff47cda4aa0fe7a11bb63c, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/2f58d38a34d64cd9891eadc9f641e5e0, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/024a867da0cf4e118019c1095fe17407, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/7e98b0fbd64d48c78b3ed54efa7daeb0, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/f8753d1b00a542b889652ff83f3b7257, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/34d4c3063d134d319370c82f642ebfe9, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/b69ad902f45749899afd5ccac22a1428, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/37c7bfcefea64727bad363f9c267f95a, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/f99604ef6fcd40ef8ffe395ea540a664, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/dc3b6d8dcead47c9a1bf8139febe8d8a, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/95fbe683c9894212984e4768d056a8e0, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/13a68634aa374946abe37fb2176bfaae, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/b6a90f0f349d47a2984663e3b3967f4e] to archive 2024-11-07T09:37:56,698 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9f1eb259a29b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T09:37:56,700 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9f1eb259a29b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/c5d0b0e8a858466a8232572debb86dfd to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/c5d0b0e8a858466a8232572debb86dfd 2024-11-07T09:37:56,701 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9f1eb259a29b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/bb491638daf2457b8c35e2b3822daea2 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/bb491638daf2457b8c35e2b3822daea2 2024-11-07T09:37:56,703 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9f1eb259a29b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/88bd9057ff244159abee084c8557ee1b to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/88bd9057ff244159abee084c8557ee1b 2024-11-07T09:37:56,704 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9f1eb259a29b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/efa6f537a1ff47cda4aa0fe7a11bb63c to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/efa6f537a1ff47cda4aa0fe7a11bb63c 2024-11-07T09:37:56,706 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9f1eb259a29b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/2f58d38a34d64cd9891eadc9f641e5e0 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/2f58d38a34d64cd9891eadc9f641e5e0 2024-11-07T09:37:56,707 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9f1eb259a29b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/024a867da0cf4e118019c1095fe17407 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/024a867da0cf4e118019c1095fe17407 2024-11-07T09:37:56,708 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9f1eb259a29b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/7e98b0fbd64d48c78b3ed54efa7daeb0 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/7e98b0fbd64d48c78b3ed54efa7daeb0 2024-11-07T09:37:56,709 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9f1eb259a29b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/f8753d1b00a542b889652ff83f3b7257 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/f8753d1b00a542b889652ff83f3b7257 2024-11-07T09:37:56,712 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9f1eb259a29b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/34d4c3063d134d319370c82f642ebfe9 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/34d4c3063d134d319370c82f642ebfe9 2024-11-07T09:37:56,714 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9f1eb259a29b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/b69ad902f45749899afd5ccac22a1428 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/b69ad902f45749899afd5ccac22a1428 2024-11-07T09:37:56,716 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9f1eb259a29b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/37c7bfcefea64727bad363f9c267f95a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/37c7bfcefea64727bad363f9c267f95a 2024-11-07T09:37:56,719 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9f1eb259a29b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/f99604ef6fcd40ef8ffe395ea540a664 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/f99604ef6fcd40ef8ffe395ea540a664 2024-11-07T09:37:56,720 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7f7127bc07c9cd4d3f409040fbdd89d3#A#compaction#378 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 65.38 MB/second 2024-11-07T09:37:56,720 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9f1eb259a29b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/dc3b6d8dcead47c9a1bf8139febe8d8a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/dc3b6d8dcead47c9a1bf8139febe8d8a 2024-11-07T09:37:56,721 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/01e0ff396e4741718a9220c6a2e65974 is 175, key is test_row_0/A:col10/1730972274931/Put/seqid=0 2024-11-07T09:37:56,722 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9f1eb259a29b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/95fbe683c9894212984e4768d056a8e0 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/95fbe683c9894212984e4768d056a8e0 2024-11-07T09:37:56,723 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9f1eb259a29b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/13a68634aa374946abe37fb2176bfaae to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/13a68634aa374946abe37fb2176bfaae 2024-11-07T09:37:56,725 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9f1eb259a29b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/b6a90f0f349d47a2984663e3b3967f4e to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/b6a90f0f349d47a2984663e3b3967f4e 2024-11-07T09:37:56,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742271_1447 (size=31345) 2024-11-07T09:37:56,743 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:56,744 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/01e0ff396e4741718a9220c6a2e65974 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/01e0ff396e4741718a9220c6a2e65974 2024-11-07T09:37:56,748 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411076460c27e4f5345f8a4cb01e5a59ef867_7f7127bc07c9cd4d3f409040fbdd89d3 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411076460c27e4f5345f8a4cb01e5a59ef867_7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:56,749 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/e3b3454c59824d81a57c439d0a539a9f, store: [table=TestAcidGuarantees family=A region=7f7127bc07c9cd4d3f409040fbdd89d3] 2024-11-07T09:37:56,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/e3b3454c59824d81a57c439d0a539a9f is 175, key is test_row_0/A:col10/1730972275111/Put/seqid=0 2024-11-07T09:37:56,751 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in 7f7127bc07c9cd4d3f409040fbdd89d3/A of 7f7127bc07c9cd4d3f409040fbdd89d3 into 01e0ff396e4741718a9220c6a2e65974(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:37:56,751 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7f7127bc07c9cd4d3f409040fbdd89d3: 2024-11-07T09:37:56,751 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., storeName=7f7127bc07c9cd4d3f409040fbdd89d3/A, priority=9, startTime=1730972276240; duration=0sec 2024-11-07T09:37:56,752 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:56,752 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f7127bc07c9cd4d3f409040fbdd89d3:A 2024-11-07T09:37:56,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742272_1448 (size=39949) 2024-11-07T09:37:56,759 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=268, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/e3b3454c59824d81a57c439d0a539a9f 2024-11-07T09:37:56,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/2a60fde087cf4e8cab4e5f2d3482b228 is 50, key is test_row_0/B:col10/1730972275111/Put/seqid=0 2024-11-07T09:37:56,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742273_1449 (size=12301) 2024-11-07T09:37:56,782 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/2a60fde087cf4e8cab4e5f2d3482b228 2024-11-07T09:37:56,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/4ad7e370594247a1b973d331658526cc is 50, key is test_row_0/C:col10/1730972275111/Put/seqid=0 2024-11-07T09:37:56,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742274_1450 (size=12301) 2024-11-07T09:37:56,800 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/4ad7e370594247a1b973d331658526cc 2024-11-07T09:37:56,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/e3b3454c59824d81a57c439d0a539a9f as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/e3b3454c59824d81a57c439d0a539a9f 2024-11-07T09:37:56,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/e3b3454c59824d81a57c439d0a539a9f, entries=200, sequenceid=268, filesize=39.0 K 2024-11-07T09:37:56,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/2a60fde087cf4e8cab4e5f2d3482b228 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/2a60fde087cf4e8cab4e5f2d3482b228 2024-11-07T09:37:56,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/2a60fde087cf4e8cab4e5f2d3482b228, entries=150, sequenceid=268, filesize=12.0 K 2024-11-07T09:37:56,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/4ad7e370594247a1b973d331658526cc as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/4ad7e370594247a1b973d331658526cc 2024-11-07T09:37:56,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-07T09:37:56,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:56,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. as already flushing 2024-11-07T09:37:56,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:56,817 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:56,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:56,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:37:56,826 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/4ad7e370594247a1b973d331658526cc, entries=150, sequenceid=268, filesize=12.0 K 2024-11-07T09:37:56,826 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 7f7127bc07c9cd4d3f409040fbdd89d3 in 536ms, sequenceid=268, compaction requested=false 2024-11-07T09:37:56,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7f7127bc07c9cd4d3f409040fbdd89d3: 2024-11-07T09:37:56,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-07T09:37:56,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:56,973 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 7f7127bc07c9cd4d3f409040fbdd89d3 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T09:37:56,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:56,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. as already flushing 2024-11-07T09:37:56,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411077193342e1a8b4ec7ad2b1e1aad49a0e0_7f7127bc07c9cd4d3f409040fbdd89d3 is 50, key is test_row_0/A:col10/1730972276308/Put/seqid=0 2024-11-07T09:37:56,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742275_1451 (size=12454) 2024-11-07T09:37:57,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:57,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49148 deadline: 1730972337062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:57,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:57,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:57,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:57,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:57,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49098 deadline: 1730972337077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:57,087 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:57,087 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:57,087 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:57,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:57,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49086 deadline: 1730972337077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:57,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:57,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:57,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:57,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-07T09:37:57,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49148 deadline: 1730972337182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:57,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:57,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:57,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:57,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:57,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49098 deadline: 1730972337193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:57,203 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:57,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:57,203 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:57,203 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:57,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49086 deadline: 1730972337193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:57,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:57,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:57,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:57,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:57,389 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411077193342e1a8b4ec7ad2b1e1aad49a0e0_7f7127bc07c9cd4d3f409040fbdd89d3 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411077193342e1a8b4ec7ad2b1e1aad49a0e0_7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:57,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/9d0914d14c32435aa73a4425ebc778db, store: [table=TestAcidGuarantees family=A region=7f7127bc07c9cd4d3f409040fbdd89d3] 2024-11-07T09:37:57,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/9d0914d14c32435aa73a4425ebc778db is 175, key is test_row_0/A:col10/1730972276308/Put/seqid=0 2024-11-07T09:37:57,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742276_1452 (size=31255) 2024-11-07T09:37:57,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:57,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49148 deadline: 1730972337402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:57,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:57,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:57,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:57,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:57,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49098 deadline: 1730972337413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:57,424 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:57,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:57,424 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:57,424 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:57,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49086 deadline: 1730972337413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:57,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:57,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:57,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:57,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:57,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49148 deadline: 1730972337712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:57,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:57,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:57,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:57,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:57,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49098 deadline: 1730972337733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:57,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:37:57,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:57,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:57,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:57,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49086 deadline: 1730972337733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:57,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:57,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:37:57,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:37:57,795 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=282, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/9d0914d14c32435aa73a4425ebc778db 2024-11-07T09:37:57,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/e3ea7aa3be5e4a459b6a2acbedbe261a is 50, key is test_row_0/B:col10/1730972276308/Put/seqid=0 2024-11-07T09:37:57,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742277_1453 (size=12301) 2024-11-07T09:37:57,811 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/e3ea7aa3be5e4a459b6a2acbedbe261a 2024-11-07T09:37:57,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/60926c2b663d428ab59c740981b14ff9 is 50, key is test_row_0/C:col10/1730972276308/Put/seqid=0 2024-11-07T09:37:57,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742278_1454 (size=12301) 2024-11-07T09:37:57,834 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/60926c2b663d428ab59c740981b14ff9 2024-11-07T09:37:57,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/9d0914d14c32435aa73a4425ebc778db as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/9d0914d14c32435aa73a4425ebc778db 2024-11-07T09:37:57,852 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/9d0914d14c32435aa73a4425ebc778db, entries=150, sequenceid=282, filesize=30.5 K 2024-11-07T09:37:57,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/e3ea7aa3be5e4a459b6a2acbedbe261a as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/e3ea7aa3be5e4a459b6a2acbedbe261a 2024-11-07T09:37:57,857 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/e3ea7aa3be5e4a459b6a2acbedbe261a, entries=150, sequenceid=282, filesize=12.0 K 2024-11-07T09:37:57,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/60926c2b663d428ab59c740981b14ff9 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/60926c2b663d428ab59c740981b14ff9 2024-11-07T09:37:57,861 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/60926c2b663d428ab59c740981b14ff9, entries=150, sequenceid=282, filesize=12.0 K 2024-11-07T09:37:57,861 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 7f7127bc07c9cd4d3f409040fbdd89d3 in 888ms, sequenceid=282, compaction requested=true 2024-11-07T09:37:57,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 7f7127bc07c9cd4d3f409040fbdd89d3: 2024-11-07T09:37:57,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:57,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-07T09:37:57,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-07T09:37:57,864 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-07T09:37:57,864 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8860 sec 2024-11-07T09:37:57,866 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 2.8910 sec 2024-11-07T09:37:57,886 INFO [Thread-1724 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1724. 2024-11-07T09:37:57,886 DEBUG [Thread-1724 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:37:57,887 DEBUG [Thread-1724 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:57,887 DEBUG [Thread-1724 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:57,887 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:37:57,888 INFO [Thread-1720 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1720. 2024-11-07T09:37:57,888 DEBUG [Thread-1720 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:37:57,888 DEBUG [Thread-1720 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:57,888 DEBUG [Thread-1720 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:57,888 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:37:57,890 INFO [Thread-1718 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1718. 2024-11-07T09:37:57,890 DEBUG [Thread-1718 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:37:57,890 DEBUG [Thread-1718 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:57,891 DEBUG [Thread-1718 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:57,891 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:37:57,895 INFO [Thread-1716 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1716. 2024-11-07T09:37:57,895 DEBUG [Thread-1716 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:37:57,895 DEBUG [Thread-1716 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:57,895 INFO [Thread-1722 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1722. 2024-11-07T09:37:57,895 DEBUG [Thread-1722 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:37:57,895 DEBUG [Thread-1722 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:57,895 DEBUG [Thread-1716 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:57,895 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:37:57,895 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:37:57,896 DEBUG [Thread-1722 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:58,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:58,234 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:37:58,234 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7f7127bc07c9cd4d3f409040fbdd89d3 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-07T09:37:58,234 INFO [Thread-1707 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1707. 2024-11-07T09:37:58,234 DEBUG [Thread-1707 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:37:58,234 DEBUG [Thread-1707 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:58,235 DEBUG [Thread-1707 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:58,235 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:37:58,240 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107896c57b102ca4cf5a8771f23df676ba2_7f7127bc07c9cd4d3f409040fbdd89d3 is 50, key is test_row_0/A:col10/1730972277075/Put/seqid=0 2024-11-07T09:37:58,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742279_1455 (size=12454) 2024-11-07T09:37:58,255 INFO [Thread-1713 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1713. 2024-11-07T09:37:58,255 DEBUG [Thread-1713 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:37:58,255 DEBUG [Thread-1713 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:58,255 DEBUG [Thread-1713 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:58,255 INFO [Thread-1711 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1711. 2024-11-07T09:37:58,255 DEBUG [Thread-1711 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:37:58,255 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:37:58,255 DEBUG [Thread-1711 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:58,255 DEBUG [Thread-1711 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:58,255 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:37:58,644 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:37:58,647 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107896c57b102ca4cf5a8771f23df676ba2_7f7127bc07c9cd4d3f409040fbdd89d3 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107896c57b102ca4cf5a8771f23df676ba2_7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:58,648 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/429c81e237ab424cb1b56ae95f1e94d5, store: [table=TestAcidGuarantees family=A region=7f7127bc07c9cd4d3f409040fbdd89d3] 2024-11-07T09:37:58,648 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/429c81e237ab424cb1b56ae95f1e94d5 is 175, key is test_row_0/A:col10/1730972277075/Put/seqid=0 2024-11-07T09:37:58,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742280_1456 (size=31255) 2024-11-07T09:37:59,052 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=309, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/429c81e237ab424cb1b56ae95f1e94d5 2024-11-07T09:37:59,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/cc071f7048be4177ac7a8e964483676f is 50, key is test_row_0/B:col10/1730972277075/Put/seqid=0 2024-11-07T09:37:59,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742281_1457 (size=12301) 2024-11-07T09:37:59,134 INFO [Thread-1705 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1705. 2024-11-07T09:37:59,134 DEBUG [Thread-1705 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:37:59,134 DEBUG [Thread-1705 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:59,134 DEBUG [Thread-1705 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:59,134 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:37:59,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-07T09:37:59,153 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:37:59,165 INFO [Thread-1709 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1709. 2024-11-07T09:37:59,165 DEBUG [Thread-1709 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:37:59,165 DEBUG [Thread-1709 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:59,165 DEBUG [Thread-1709 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:59,165 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:37:59,165 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-07T09:37:59,165 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 35 2024-11-07T09:37:59,165 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-11-07T09:37:59,165 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 33 2024-11-07T09:37:59,165 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-07T09:37:59,165 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-11-07T09:37:59,165 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-07T09:37:59,165 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-07T09:37:59,165 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 974 2024-11-07T09:37:59,165 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2921 rows 2024-11-07T09:37:59,165 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 977 2024-11-07T09:37:59,165 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2930 rows 2024-11-07T09:37:59,165 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 972 2024-11-07T09:37:59,165 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2916 rows 2024-11-07T09:37:59,165 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 984 2024-11-07T09:37:59,165 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2952 rows 2024-11-07T09:37:59,165 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 981 2024-11-07T09:37:59,165 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2941 rows 2024-11-07T09:37:59,165 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T09:37:59,166 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T09:37:59,166 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:59,166 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:37:59,166 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:37:59,167 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-07T09:37:59,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-07T09:37:59,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-07T09:37:59,169 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730972279169"}]},"ts":"1730972279169"} 2024-11-07T09:37:59,171 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-07T09:37:59,171 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-07T09:37:59,171 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T09:37:59,173 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7f7127bc07c9cd4d3f409040fbdd89d3, UNASSIGN}] 2024-11-07T09:37:59,173 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7f7127bc07c9cd4d3f409040fbdd89d3, UNASSIGN 2024-11-07T09:37:59,174 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=7f7127bc07c9cd4d3f409040fbdd89d3, regionState=CLOSING, regionLocation=9f1eb259a29b,42895,1730972153970 2024-11-07T09:37:59,175 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7f7127bc07c9cd4d3f409040fbdd89d3, UNASSIGN because future has completed 2024-11-07T09:37:59,175 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T09:37:59,175 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970}] 2024-11-07T09:37:59,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-07T09:37:59,327 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] handler.UnassignRegionHandler(122): Close 7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:37:59,327 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T09:37:59,327 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1722): Closing 7f7127bc07c9cd4d3f409040fbdd89d3, disabling compactions & flushes 2024-11-07T09:37:59,328 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:59,462 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/cc071f7048be4177ac7a8e964483676f 2024-11-07T09:37:59,469 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/f27ac238653d49ec822861dbb05bd1b2 is 50, key is test_row_0/C:col10/1730972277075/Put/seqid=0 2024-11-07T09:37:59,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742282_1458 (size=12301) 2024-11-07T09:37:59,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-07T09:37:59,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-07T09:37:59,872 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/f27ac238653d49ec822861dbb05bd1b2 2024-11-07T09:37:59,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/429c81e237ab424cb1b56ae95f1e94d5 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/429c81e237ab424cb1b56ae95f1e94d5 2024-11-07T09:37:59,879 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/429c81e237ab424cb1b56ae95f1e94d5, entries=150, sequenceid=309, filesize=30.5 K 2024-11-07T09:37:59,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/cc071f7048be4177ac7a8e964483676f as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/cc071f7048be4177ac7a8e964483676f 2024-11-07T09:37:59,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/cc071f7048be4177ac7a8e964483676f, entries=150, sequenceid=309, filesize=12.0 K 2024-11-07T09:37:59,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/f27ac238653d49ec822861dbb05bd1b2 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/f27ac238653d49ec822861dbb05bd1b2 2024-11-07T09:37:59,885 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/f27ac238653d49ec822861dbb05bd1b2, entries=150, sequenceid=309, filesize=12.0 K 2024-11-07T09:37:59,886 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=26.84 KB/27480 for 7f7127bc07c9cd4d3f409040fbdd89d3 in 1652ms, sequenceid=309, compaction requested=true 2024-11-07T09:37:59,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7f7127bc07c9cd4d3f409040fbdd89d3: 2024-11-07T09:37:59,886 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:59,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f7127bc07c9cd4d3f409040fbdd89d3:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T09:37:59,886 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:59,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:59,886 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. because compaction request was cancelled 2024-11-07T09:37:59,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f7127bc07c9cd4d3f409040fbdd89d3:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T09:37:59,886 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. after waiting 0 ms 2024-11-07T09:37:59,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:37:59,886 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f7127bc07c9cd4d3f409040fbdd89d3:A 2024-11-07T09:37:59,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f7127bc07c9cd4d3f409040fbdd89d3:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:37:59,886 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:37:59,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:37:59,886 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. because compaction request was cancelled 2024-11-07T09:37:59,886 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. because compaction request was cancelled 2024-11-07T09:37:59,886 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f7127bc07c9cd4d3f409040fbdd89d3:C 2024-11-07T09:37:59,886 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f7127bc07c9cd4d3f409040fbdd89d3:B 2024-11-07T09:37:59,886 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(2902): Flushing 7f7127bc07c9cd4d3f409040fbdd89d3 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-07T09:37:59,891 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107200b5c3f2ecf41f8a75c9e4d77524996_7f7127bc07c9cd4d3f409040fbdd89d3 is 50, key is test_row_0/A:col10/1730972279133/Put/seqid=0 2024-11-07T09:37:59,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742283_1459 (size=12454) 2024-11-07T09:38:00,295 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:00,299 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107200b5c3f2ecf41f8a75c9e4d77524996_7f7127bc07c9cd4d3f409040fbdd89d3 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107200b5c3f2ecf41f8a75c9e4d77524996_7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:38:00,299 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/16526e2842b54a48a3c32213b3717735, store: [table=TestAcidGuarantees family=A region=7f7127bc07c9cd4d3f409040fbdd89d3] 2024-11-07T09:38:00,300 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/16526e2842b54a48a3c32213b3717735 is 175, key is test_row_0/A:col10/1730972279133/Put/seqid=0 2024-11-07T09:38:00,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742284_1460 (size=31255) 2024-11-07T09:38:00,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-07T09:38:00,704 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=316, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/16526e2842b54a48a3c32213b3717735 2024-11-07T09:38:00,710 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/3544951afcd5462ebb0a0d17e54f5eba is 50, key is test_row_0/B:col10/1730972279133/Put/seqid=0 2024-11-07T09:38:00,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742285_1461 (size=12301) 2024-11-07T09:38:01,118 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/3544951afcd5462ebb0a0d17e54f5eba 2024-11-07T09:38:01,123 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/723333cc851645dfb151ea46fb7abf25 is 50, key is test_row_0/C:col10/1730972279133/Put/seqid=0 2024-11-07T09:38:01,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742286_1462 (size=12301) 2024-11-07T09:38:01,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-07T09:38:01,527 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/723333cc851645dfb151ea46fb7abf25 2024-11-07T09:38:01,530 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/A/16526e2842b54a48a3c32213b3717735 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/16526e2842b54a48a3c32213b3717735 2024-11-07T09:38:01,533 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/16526e2842b54a48a3c32213b3717735, entries=150, sequenceid=316, filesize=30.5 K 2024-11-07T09:38:01,534 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/B/3544951afcd5462ebb0a0d17e54f5eba as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/3544951afcd5462ebb0a0d17e54f5eba 2024-11-07T09:38:01,536 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/3544951afcd5462ebb0a0d17e54f5eba, entries=150, sequenceid=316, filesize=12.0 K 2024-11-07T09:38:01,537 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/.tmp/C/723333cc851645dfb151ea46fb7abf25 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/723333cc851645dfb151ea46fb7abf25 2024-11-07T09:38:01,540 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/723333cc851645dfb151ea46fb7abf25, entries=150, sequenceid=316, filesize=12.0 K 2024-11-07T09:38:01,540 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 7f7127bc07c9cd4d3f409040fbdd89d3 in 1654ms, sequenceid=316, compaction requested=true 2024-11-07T09:38:01,541 DEBUG [StoreCloser-TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/e85fe8674a744497b88d0dfa62f96685, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/e0482d64ea8b462ea5262df9edee7153, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/d24b2c29e349497e92907e822e618840, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/3606f7d70d0f4509aded546ad685a6b3, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/f9790cc940e34a1688a368e023ac45a3, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/5af75993776648d3819f1a6b869e7b46, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/42d843a9bfd14132afdf73d10a68750e] to archive 2024-11-07T09:38:01,541 DEBUG [StoreCloser-TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T09:38:01,543 DEBUG [StoreCloser-TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/e85fe8674a744497b88d0dfa62f96685 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/e85fe8674a744497b88d0dfa62f96685 2024-11-07T09:38:01,544 DEBUG [StoreCloser-TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/e0482d64ea8b462ea5262df9edee7153 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/e0482d64ea8b462ea5262df9edee7153 2024-11-07T09:38:01,545 DEBUG [StoreCloser-TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/d24b2c29e349497e92907e822e618840 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/d24b2c29e349497e92907e822e618840 2024-11-07T09:38:01,545 DEBUG [StoreCloser-TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/3606f7d70d0f4509aded546ad685a6b3 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/3606f7d70d0f4509aded546ad685a6b3 2024-11-07T09:38:01,546 DEBUG [StoreCloser-TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/f9790cc940e34a1688a368e023ac45a3 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/f9790cc940e34a1688a368e023ac45a3 2024-11-07T09:38:01,547 DEBUG [StoreCloser-TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/5af75993776648d3819f1a6b869e7b46 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/5af75993776648d3819f1a6b869e7b46 2024-11-07T09:38:01,548 DEBUG [StoreCloser-TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/42d843a9bfd14132afdf73d10a68750e to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/42d843a9bfd14132afdf73d10a68750e 2024-11-07T09:38:01,553 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/recovered.edits/319.seqid, newMaxSeqId=319, maxSeqId=4 2024-11-07T09:38:01,554 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. 2024-11-07T09:38:01,554 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1676): Region close journal for 7f7127bc07c9cd4d3f409040fbdd89d3: Waiting for close lock at 1730972279327Running coprocessor pre-close hooks at 1730972279327Disabling compacts and flushes for region at 1730972279327Disabling writes for close at 1730972279886 (+559 ms)Obtaining lock to block concurrent updates at 1730972279886Preparing flush snapshotting stores in 7f7127bc07c9cd4d3f409040fbdd89d3 at 1730972279886Finished memstore snapshotting TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1730972279886Flushing stores of TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3. at 1730972279887 (+1 ms)Flushing 7f7127bc07c9cd4d3f409040fbdd89d3/A: creating writer at 1730972279887Flushing mob file 7f7127bc07c9cd4d3f409040fbdd89d3/A: appending metadata at 1730972279891 (+4 ms)Flushing mob file 7f7127bc07c9cd4d3f409040fbdd89d3/A: closing flushed file at 1730972279891Flushing 7f7127bc07c9cd4d3f409040fbdd89d3/A: appending metadata at 1730972280299 (+408 ms)Flushing 7f7127bc07c9cd4d3f409040fbdd89d3/A: closing flushed file at 1730972280299Flushing 7f7127bc07c9cd4d3f409040fbdd89d3/B: creating writer at 1730972280707 (+408 ms)Flushing 7f7127bc07c9cd4d3f409040fbdd89d3/B: appending metadata at 1730972280709 (+2 ms)Flushing 7f7127bc07c9cd4d3f409040fbdd89d3/B: closing flushed file at 1730972280709Flushing 7f7127bc07c9cd4d3f409040fbdd89d3/C: creating writer at 1730972281121 (+412 ms)Flushing 7f7127bc07c9cd4d3f409040fbdd89d3/C: appending metadata at 1730972281123 (+2 ms)Flushing 7f7127bc07c9cd4d3f409040fbdd89d3/C: closing flushed file at 1730972281123Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@452222ac: reopening flushed file at 1730972281530 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4461b751: reopening flushed file at 1730972281533 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3265a30: reopening flushed file at 1730972281536 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 7f7127bc07c9cd4d3f409040fbdd89d3 in 1654ms, sequenceid=316, compaction requested=true at 1730972281540 (+4 ms)Writing region close event to WAL at 1730972281551 (+11 ms)Running coprocessor post-close hooks at 1730972281554 (+3 ms)Closed at 1730972281554 2024-11-07T09:38:01,556 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] handler.UnassignRegionHandler(157): Closed 7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:38:01,556 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=7f7127bc07c9cd4d3f409040fbdd89d3, regionState=CLOSED 2024-11-07T09:38:01,557 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=126, ppid=125, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 because future has completed 2024-11-07T09:38:01,561 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-07T09:38:01,561 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; CloseRegionProcedure 7f7127bc07c9cd4d3f409040fbdd89d3, server=9f1eb259a29b,42895,1730972153970 in 2.3830 sec 2024-11-07T09:38:01,562 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=125, resume processing ppid=124 2024-11-07T09:38:01,562 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, ppid=124, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7f7127bc07c9cd4d3f409040fbdd89d3, UNASSIGN in 2.3890 sec 2024-11-07T09:38:01,564 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-07T09:38:01,564 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.3920 sec 2024-11-07T09:38:01,565 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730972281565"}]},"ts":"1730972281565"} 2024-11-07T09:38:01,567 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-07T09:38:01,567 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-07T09:38:01,568 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.4000 sec 2024-11-07T09:38:03,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-07T09:38:03,323 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:38:03,324 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-07T09:38:03,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T09:38:03,326 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=127, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T09:38:03,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-07T09:38:03,326 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=127, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T09:38:03,329 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:38:03,331 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A, FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B, FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C, FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/recovered.edits] 2024-11-07T09:38:03,333 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/01e0ff396e4741718a9220c6a2e65974 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/01e0ff396e4741718a9220c6a2e65974 2024-11-07T09:38:03,334 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/16526e2842b54a48a3c32213b3717735 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/16526e2842b54a48a3c32213b3717735 2024-11-07T09:38:03,334 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/429c81e237ab424cb1b56ae95f1e94d5 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/429c81e237ab424cb1b56ae95f1e94d5 2024-11-07T09:38:03,335 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/9d0914d14c32435aa73a4425ebc778db to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/9d0914d14c32435aa73a4425ebc778db 2024-11-07T09:38:03,337 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/e3b3454c59824d81a57c439d0a539a9f to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/A/e3b3454c59824d81a57c439d0a539a9f 2024-11-07T09:38:03,339 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/2a60fde087cf4e8cab4e5f2d3482b228 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/2a60fde087cf4e8cab4e5f2d3482b228 2024-11-07T09:38:03,340 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/3544951afcd5462ebb0a0d17e54f5eba to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/3544951afcd5462ebb0a0d17e54f5eba 2024-11-07T09:38:03,340 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/98ecf4942b1d45a2954168f26dd43cfd to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/98ecf4942b1d45a2954168f26dd43cfd 2024-11-07T09:38:03,341 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/cc071f7048be4177ac7a8e964483676f to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/cc071f7048be4177ac7a8e964483676f 2024-11-07T09:38:03,342 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/e3ea7aa3be5e4a459b6a2acbedbe261a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/B/e3ea7aa3be5e4a459b6a2acbedbe261a 2024-11-07T09:38:03,344 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/4ad7e370594247a1b973d331658526cc to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/4ad7e370594247a1b973d331658526cc 2024-11-07T09:38:03,344 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/60926c2b663d428ab59c740981b14ff9 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/60926c2b663d428ab59c740981b14ff9 2024-11-07T09:38:03,345 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/723333cc851645dfb151ea46fb7abf25 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/723333cc851645dfb151ea46fb7abf25 2024-11-07T09:38:03,346 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/d8ae26052b2d47cfbc04cd56a219ba9b to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/d8ae26052b2d47cfbc04cd56a219ba9b 2024-11-07T09:38:03,347 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/f27ac238653d49ec822861dbb05bd1b2 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/C/f27ac238653d49ec822861dbb05bd1b2 2024-11-07T09:38:03,349 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/recovered.edits/319.seqid to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3/recovered.edits/319.seqid 2024-11-07T09:38:03,349 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:38:03,349 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-07T09:38:03,350 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-07T09:38:03,350 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-07T09:38:03,352 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070e232ac73d7b42c3855d0cf89c3ed2fa_7f7127bc07c9cd4d3f409040fbdd89d3 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070e232ac73d7b42c3855d0cf89c3ed2fa_7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:38:03,353 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107161dd5994eb7430bbfc03e2952511750_7f7127bc07c9cd4d3f409040fbdd89d3 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107161dd5994eb7430bbfc03e2952511750_7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:38:03,354 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107200b5c3f2ecf41f8a75c9e4d77524996_7f7127bc07c9cd4d3f409040fbdd89d3 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107200b5c3f2ecf41f8a75c9e4d77524996_7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:38:03,355 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110724e3ac8e2f6d4228ba3c9c652866e90f_7f7127bc07c9cd4d3f409040fbdd89d3 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110724e3ac8e2f6d4228ba3c9c652866e90f_7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:38:03,356 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107588e7610275943e3afa7e4247f585e54_7f7127bc07c9cd4d3f409040fbdd89d3 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107588e7610275943e3afa7e4247f585e54_7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:38:03,357 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411076460c27e4f5345f8a4cb01e5a59ef867_7f7127bc07c9cd4d3f409040fbdd89d3 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411076460c27e4f5345f8a4cb01e5a59ef867_7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:38:03,358 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411076ac869c535754f98887c2675e7563ed5_7f7127bc07c9cd4d3f409040fbdd89d3 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411076ac869c535754f98887c2675e7563ed5_7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:38:03,359 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411077193342e1a8b4ec7ad2b1e1aad49a0e0_7f7127bc07c9cd4d3f409040fbdd89d3 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411077193342e1a8b4ec7ad2b1e1aad49a0e0_7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:38:03,360 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110774b88e95c6b74adc809b5a12848d390c_7f7127bc07c9cd4d3f409040fbdd89d3 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110774b88e95c6b74adc809b5a12848d390c_7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:38:03,361 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107896c57b102ca4cf5a8771f23df676ba2_7f7127bc07c9cd4d3f409040fbdd89d3 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107896c57b102ca4cf5a8771f23df676ba2_7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:38:03,362 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107ad6b3289aae943299bd3a86355cfcc8b_7f7127bc07c9cd4d3f409040fbdd89d3 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107ad6b3289aae943299bd3a86355cfcc8b_7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:38:03,363 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107d02e46862f954245a0acd69dbfca48c7_7f7127bc07c9cd4d3f409040fbdd89d3 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107d02e46862f954245a0acd69dbfca48c7_7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:38:03,364 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107d38388ca23734112ac9357623ff668ee_7f7127bc07c9cd4d3f409040fbdd89d3 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107d38388ca23734112ac9357623ff668ee_7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:38:03,365 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107db4386e1658e41d9bbd804f3ef732ef8_7f7127bc07c9cd4d3f409040fbdd89d3 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107db4386e1658e41d9bbd804f3ef732ef8_7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:38:03,366 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107def42aa69ed4417bb5c45b698b330ad3_7f7127bc07c9cd4d3f409040fbdd89d3 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107def42aa69ed4417bb5c45b698b330ad3_7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:38:03,367 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107e7b1b11c680b4b5a9b3b9fba23742d27_7f7127bc07c9cd4d3f409040fbdd89d3 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107e7b1b11c680b4b5a9b3b9fba23742d27_7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:38:03,368 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107e9b0c1c8a0614ffb876665f7971e996c_7f7127bc07c9cd4d3f409040fbdd89d3 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107e9b0c1c8a0614ffb876665f7971e996c_7f7127bc07c9cd4d3f409040fbdd89d3 2024-11-07T09:38:03,368 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-07T09:38:03,370 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=127, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T09:38:03,373 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-07T09:38:03,374 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-07T09:38:03,375 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=127, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T09:38:03,375 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-07T09:38:03,375 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1730972283375"}]},"ts":"9223372036854775807"} 2024-11-07T09:38:03,377 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-07T09:38:03,377 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 7f7127bc07c9cd4d3f409040fbdd89d3, NAME => 'TestAcidGuarantees,,1730972254784.7f7127bc07c9cd4d3f409040fbdd89d3.', STARTKEY => '', ENDKEY => ''}] 2024-11-07T09:38:03,377 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-07T09:38:03,377 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1730972283377"}]},"ts":"9223372036854775807"} 2024-11-07T09:38:03,378 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-07T09:38:03,379 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=127, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T09:38:03,380 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 55 msec 2024-11-07T09:38:03,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-07T09:38:03,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-07T09:38:03,433 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:38:03,446 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=242 (was 237) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-11 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_COMPACTED_FILES_DISCHARGER-regionserver/9f1eb259a29b:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2090188048_22 at /127.0.0.1:56644 [Waiting for operation #277] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-10 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_651746630_22 at /127.0.0.1:52020 [Waiting for operation #418] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_651746630_22 at /127.0.0.1:56646 [Waiting for operation #268] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2090188048_22 at /127.0.0.1:56618 [Waiting for operation #290] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=372 (was 355) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4352 (was 4466) 2024-11-07T09:38:03,457 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=242, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=372, ProcessCount=11, AvailableMemoryMB=4352 2024-11-07T09:38:03,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T09:38:03,459 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T09:38:03,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=128, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-07T09:38:03,460 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T09:38:03,460 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:03,460 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 128 2024-11-07T09:38:03,461 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T09:38:03,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-07T09:38:03,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742287_1463 (size=959) 2024-11-07T09:38:03,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-07T09:38:03,713 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-07T09:38:03,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-07T09:38:03,867 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260 2024-11-07T09:38:03,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742288_1464 (size=53) 2024-11-07T09:38:03,873 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T09:38:03,873 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 57c7d380eebbfff219ec5446401d9b7d, disabling compactions & flushes 2024-11-07T09:38:03,873 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:03,873 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:03,873 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. after waiting 0 ms 2024-11-07T09:38:03,873 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:03,873 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:03,873 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 57c7d380eebbfff219ec5446401d9b7d: Waiting for close lock at 1730972283873Disabling compacts and flushes for region at 1730972283873Disabling writes for close at 1730972283873Writing region close event to WAL at 1730972283873Closed at 1730972283873 2024-11-07T09:38:03,874 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T09:38:03,874 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1730972283874"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730972283874"}]},"ts":"1730972283874"} 2024-11-07T09:38:03,875 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-07T09:38:03,876 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T09:38:03,876 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730972283876"}]},"ts":"1730972283876"} 2024-11-07T09:38:03,877 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-07T09:38:03,878 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=57c7d380eebbfff219ec5446401d9b7d, ASSIGN}] 2024-11-07T09:38:03,878 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=57c7d380eebbfff219ec5446401d9b7d, ASSIGN 2024-11-07T09:38:03,879 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=57c7d380eebbfff219ec5446401d9b7d, ASSIGN; state=OFFLINE, location=9f1eb259a29b,42895,1730972153970; forceNewPlan=false, retain=false 2024-11-07T09:38:04,030 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=57c7d380eebbfff219ec5446401d9b7d, regionState=OPENING, regionLocation=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:04,031 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=57c7d380eebbfff219ec5446401d9b7d, ASSIGN because future has completed 2024-11-07T09:38:04,032 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; OpenRegionProcedure 57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970}] 2024-11-07T09:38:04,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-07T09:38:04,186 INFO [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:04,186 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(7752): Opening region: {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} 2024-11-07T09:38:04,187 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 57c7d380eebbfff219ec5446401d9b7d 2024-11-07T09:38:04,187 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T09:38:04,187 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(7794): checking encryption for 57c7d380eebbfff219ec5446401d9b7d 2024-11-07T09:38:04,187 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(7797): checking classloading for 57c7d380eebbfff219ec5446401d9b7d 2024-11-07T09:38:04,188 INFO [StoreOpener-57c7d380eebbfff219ec5446401d9b7d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 57c7d380eebbfff219ec5446401d9b7d 2024-11-07T09:38:04,189 INFO [StoreOpener-57c7d380eebbfff219ec5446401d9b7d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 57c7d380eebbfff219ec5446401d9b7d columnFamilyName A 2024-11-07T09:38:04,189 DEBUG [StoreOpener-57c7d380eebbfff219ec5446401d9b7d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:04,189 INFO [StoreOpener-57c7d380eebbfff219ec5446401d9b7d-1 {}] regionserver.HStore(327): Store=57c7d380eebbfff219ec5446401d9b7d/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T09:38:04,190 INFO [StoreOpener-57c7d380eebbfff219ec5446401d9b7d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 57c7d380eebbfff219ec5446401d9b7d 2024-11-07T09:38:04,191 INFO [StoreOpener-57c7d380eebbfff219ec5446401d9b7d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 57c7d380eebbfff219ec5446401d9b7d columnFamilyName B 2024-11-07T09:38:04,191 DEBUG [StoreOpener-57c7d380eebbfff219ec5446401d9b7d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:04,191 INFO [StoreOpener-57c7d380eebbfff219ec5446401d9b7d-1 {}] regionserver.HStore(327): Store=57c7d380eebbfff219ec5446401d9b7d/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T09:38:04,191 INFO [StoreOpener-57c7d380eebbfff219ec5446401d9b7d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 57c7d380eebbfff219ec5446401d9b7d 2024-11-07T09:38:04,192 INFO [StoreOpener-57c7d380eebbfff219ec5446401d9b7d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 57c7d380eebbfff219ec5446401d9b7d columnFamilyName C 2024-11-07T09:38:04,192 DEBUG [StoreOpener-57c7d380eebbfff219ec5446401d9b7d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:04,192 INFO [StoreOpener-57c7d380eebbfff219ec5446401d9b7d-1 {}] regionserver.HStore(327): Store=57c7d380eebbfff219ec5446401d9b7d/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T09:38:04,192 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1038): replaying wal for 57c7d380eebbfff219ec5446401d9b7d 2024-11-07T09:38:04,193 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d 2024-11-07T09:38:04,193 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d 2024-11-07T09:38:04,194 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1048): stopping wal replay for 57c7d380eebbfff219ec5446401d9b7d 2024-11-07T09:38:04,194 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1060): Cleaning up temporary data for 57c7d380eebbfff219ec5446401d9b7d 2024-11-07T09:38:04,194 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T09:38:04,195 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1093): writing seq id for 57c7d380eebbfff219ec5446401d9b7d 2024-11-07T09:38:04,197 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T09:38:04,197 INFO [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1114): Opened 57c7d380eebbfff219ec5446401d9b7d; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59547253, jitterRate=-0.11267678439617157}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T09:38:04,197 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 57c7d380eebbfff219ec5446401d9b7d 2024-11-07T09:38:04,198 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1006): Region open journal for 57c7d380eebbfff219ec5446401d9b7d: Running coprocessor pre-open hook at 1730972284187Writing region info on filesystem at 1730972284187Initializing all the Stores at 1730972284188 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730972284188Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730972284188Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730972284188Cleaning up temporary data from old regions at 1730972284194 (+6 ms)Running coprocessor post-open hooks at 1730972284197 (+3 ms)Region opened successfully at 1730972284198 (+1 ms) 2024-11-07T09:38:04,199 INFO [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., pid=130, masterSystemTime=1730972284183 2024-11-07T09:38:04,201 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:04,201 INFO [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:04,201 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=57c7d380eebbfff219ec5446401d9b7d, regionState=OPEN, openSeqNum=2, regionLocation=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:04,202 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=130, ppid=129, state=RUNNABLE, hasLock=false; OpenRegionProcedure 57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 because future has completed 2024-11-07T09:38:04,205 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-07T09:38:04,205 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; OpenRegionProcedure 57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 in 171 msec 2024-11-07T09:38:04,207 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=129, resume processing ppid=128 2024-11-07T09:38:04,207 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, ppid=128, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=57c7d380eebbfff219ec5446401d9b7d, ASSIGN in 327 msec 2024-11-07T09:38:04,207 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T09:38:04,207 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730972284207"}]},"ts":"1730972284207"} 2024-11-07T09:38:04,209 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-07T09:38:04,209 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T09:38:04,211 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 750 msec 2024-11-07T09:38:04,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-07T09:38:04,593 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:38:04,594 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@558db58b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:04,594 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:38:04,594 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:38:04,595 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:38:04,595 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:38:04,595 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:38:04,595 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21b25ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:04,595 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:38:04,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:38:04,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:04,597 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48106, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:38:04,597 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e0c4fcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:04,598 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:38:04,598 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:38:04,598 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:38:04,599 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38272, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:38:04,600 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T09:38:04,600 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 9f1eb259a29b,41689,1730972153310 2024-11-07T09:38:04,600 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@451fa918 2024-11-07T09:38:04,600 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T09:38:04,601 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48108, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T09:38:04,602 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7631696d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:04,602 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:38:04,602 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:38:04,603 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:38:04,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:38:04,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:38:04,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@545a3272, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:04,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:38:04,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:38:04,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:04,604 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48114, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:38:04,605 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c5af836, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:04,605 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@126b8236, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:04,605 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:38:04,605 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:38:04,606 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:38:04,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:38:04,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:38:04,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4226c197, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:04,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:38:04,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:38:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:04,607 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48124, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:38:04,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e9e1fe2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:04,608 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ea00808, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:04,608 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:38:04,608 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:38:04,609 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:38:04,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:38:04,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:38:04,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26d3aecb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:04,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:38:04,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:38:04,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:04,610 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48132, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:38:04,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18977585, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:04,611 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1589e05f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:04,611 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:38:04,611 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:38:04,611 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:38:04,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:38:04,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:38:04,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49e88a71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:04,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:38:04,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:38:04,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:04,612 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48150, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:38:04,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d4a3a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:04,613 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e46d68a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:04,613 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:38:04,613 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:38:04,614 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:38:04,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:38:04,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:38:04,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c5bb653, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:04,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:38:04,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:38:04,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:04,615 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48168, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:38:04,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e7a068f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:04,616 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45b23f2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:04,616 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:38:04,616 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:38:04,616 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:38:04,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:38:04,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:38:04,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@88bc066, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:04,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:38:04,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:38:04,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:04,618 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48182, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:38:04,618 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35c4495, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:04,618 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c7d791a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:04,618 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:38:04,619 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:38:04,619 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:38:04,619 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:38:04,619 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:38:04,619 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4914fd93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:04,619 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:38:04,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:38:04,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:04,621 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48202, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:38:04,621 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5aee35ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:04,621 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57b3a94b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:04,621 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:38:04,621 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:38:04,622 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:38:04,622 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:38:04,622 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:38:04,622 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@411f4cc2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:04,622 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:38:04,622 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:38:04,622 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:04,623 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48216, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:38:04,623 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@303c69c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:04,624 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c68a1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:04,624 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:38:04,624 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:38:04,624 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:38:04,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:38:04,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:38:04,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@364a5a43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:04,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:38:04,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:38:04,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:04,625 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48222, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:38:04,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79d3299, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:04,626 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a5c6054, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:04,626 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:38:04,626 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:38:04,627 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:38:04,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:38:04,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:38:04,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@635cb281, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:04,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:38:04,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:38:04,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:04,628 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48230, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:38:04,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@188da8f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:04,629 DEBUG [Thread-2081 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:38:04,629 DEBUG [Thread-2083 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:38:04,629 DEBUG [Thread-2092 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:38:04,629 DEBUG [Thread-2090 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:38:04,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:38:04,630 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:38:04,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:38:04,630 DEBUG [Thread-2096 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:38:04,630 DEBUG [Thread-2085 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:38:04,630 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:38:04,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:38:04,630 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:38:04,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:38:04,630 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:38:04,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:38:04,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:38:04,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:38:04,631 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38288, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:38:04,631 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38310, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:38:04,631 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T09:38:04,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:38:04,632 DEBUG [Thread-2087 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:38:04,632 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38318, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:38:04,632 DEBUG [Thread-2098 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:38:04,632 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38294, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:38:04,633 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38328, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:38:04,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:38:04,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:38:04,633 DEBUG [Thread-2094 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:38:04,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-07T09:38:04,634 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38344, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:38:04,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-07T09:38:04,634 DEBUG [Thread-2079 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:38:04,634 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:38:04,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2] 2024-11-07T09:38:04,635 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:38:04,635 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T09:38:04,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:38:04,636 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:38:04,636 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38354, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:38:04,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2] 2024-11-07T09:38:04,636 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T09:38:04,636 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T09:38:04,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2] 2024-11-07T09:38:04,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2] 2024-11-07T09:38:04,638 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38332, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:38:04,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2] 2024-11-07T09:38:04,638 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38368, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:38:04,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:38:04,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:38:04,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2] 2024-11-07T09:38:04,639 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38374, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:38:04,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2] 2024-11-07T09:38:04,639 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2] 2024-11-07T09:38:04,641 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2] 2024-11-07T09:38:04,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2] 2024-11-07T09:38:04,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 57c7d380eebbfff219ec5446401d9b7d 2024-11-07T09:38:04,652 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:38:04,652 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 57c7d380eebbfff219ec5446401d9b7d 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-07T09:38:04,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/c9071ab0d6f44eb0a9904da1a3d163c4 is 50, key is test_row_1/A:col10/1730972284651/Put/seqid=0 2024-11-07T09:38:04,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742289_1465 (size=16677) 2024-11-07T09:38:04,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:04,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972344692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:04,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:04,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:04,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:04,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:04,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38374 deadline: 1730972344696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:04,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:04,708 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:04,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972344697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:04,709 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:04,709 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:04,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:04,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972344697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:04,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:04,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:04,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:04,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:04,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:04,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:04,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:04,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38332 deadline: 1730972344704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:04,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:04,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:04,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:04,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-07T09:38:04,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-07T09:38:04,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:04,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. as already flushing 2024-11-07T09:38:04,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:04,790 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:04,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:04,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:04,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:04,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972344813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:04,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:04,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38374 deadline: 1730972344813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:04,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:04,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:04,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:04,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:04,814 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:04,814 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:04,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972344813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:04,814 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:04,815 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:04,815 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:04,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:04,815 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:04,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972344813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:04,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:04,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:04,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:04,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:04,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38332 deadline: 1730972344824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:04,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:04,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:04,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:04,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-07T09:38:04,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:04,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. as already flushing 2024-11-07T09:38:04,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:04,945 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:04,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:04,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:04,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-07T09:38:05,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:05,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972345023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:05,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:05,030 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:05,030 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:05,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972345024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:05,030 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:05,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:05,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38374 deadline: 1730972345024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:05,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:05,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972345024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:05,031 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:05,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:05,031 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:05,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:05,031 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:05,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:05,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:05,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:05,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:05,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:05,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38332 deadline: 1730972345043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:05,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:05,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:05,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:05,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-07T09:38:05,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:05,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. as already flushing 2024-11-07T09:38:05,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:05,100 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:05,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:05,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:05,103 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/c9071ab0d6f44eb0a9904da1a3d163c4 2024-11-07T09:38:05,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/5aa7244bae9c4cebac41dd6a47a1ac90 is 50, key is test_row_1/B:col10/1730972284651/Put/seqid=0 2024-11-07T09:38:05,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742290_1466 (size=9657) 2024-11-07T09:38:05,137 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/5aa7244bae9c4cebac41dd6a47a1ac90 2024-11-07T09:38:05,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/76b11018c64e4d41a6fbe8c677fb9b56 is 50, key is test_row_1/C:col10/1730972284651/Put/seqid=0 2024-11-07T09:38:05,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742291_1467 (size=9657) 2024-11-07T09:38:05,254 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-07T09:38:05,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:05,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. as already flushing 2024-11-07T09:38:05,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:05,254 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:05,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:05,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:05,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-07T09:38:05,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:05,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972345333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:05,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:05,335 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:05,336 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:05,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38374 deadline: 1730972345333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:05,336 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:05,336 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:05,336 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:05,336 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:05,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:05,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972345333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:05,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:05,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:05,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:05,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:05,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972345342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:05,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:05,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:05,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:05,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:05,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38332 deadline: 1730972345363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:05,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:05,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:05,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:05,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-07T09:38:05,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:05,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. as already flushing 2024-11-07T09:38:05,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:05,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:05,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:05,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:05,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-07T09:38:05,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:05,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. as already flushing 2024-11-07T09:38:05,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:05,560 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:05,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:05,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:05,577 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/76b11018c64e4d41a6fbe8c677fb9b56 2024-11-07T09:38:05,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/c9071ab0d6f44eb0a9904da1a3d163c4 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/c9071ab0d6f44eb0a9904da1a3d163c4 2024-11-07T09:38:05,584 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/c9071ab0d6f44eb0a9904da1a3d163c4, entries=250, sequenceid=14, filesize=16.3 K 2024-11-07T09:38:05,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/5aa7244bae9c4cebac41dd6a47a1ac90 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/5aa7244bae9c4cebac41dd6a47a1ac90 2024-11-07T09:38:05,588 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/5aa7244bae9c4cebac41dd6a47a1ac90, entries=100, sequenceid=14, filesize=9.4 K 2024-11-07T09:38:05,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/76b11018c64e4d41a6fbe8c677fb9b56 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/76b11018c64e4d41a6fbe8c677fb9b56 2024-11-07T09:38:05,592 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/76b11018c64e4d41a6fbe8c677fb9b56, entries=100, sequenceid=14, filesize=9.4 K 2024-11-07T09:38:05,592 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 57c7d380eebbfff219ec5446401d9b7d in 940ms, sequenceid=14, compaction requested=false 2024-11-07T09:38:05,592 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-07T09:38:05,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 57c7d380eebbfff219ec5446401d9b7d: 2024-11-07T09:38:05,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-07T09:38:05,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:05,713 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 57c7d380eebbfff219ec5446401d9b7d 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-07T09:38:05,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/1d23c12a75334a289a845eead6958492 is 50, key is test_row_0/A:col10/1730972284690/Put/seqid=0 2024-11-07T09:38:05,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742292_1468 (size=12001) 2024-11-07T09:38:05,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-07T09:38:05,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 57c7d380eebbfff219ec5446401d9b7d 2024-11-07T09:38:05,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. as already flushing 2024-11-07T09:38:05,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:05,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972345863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:05,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:05,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:05,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:05,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38374 deadline: 1730972345871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:05,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:05,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:05,874 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:05,874 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:05,874 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:05,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972345871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:05,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:05,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:05,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:05,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:05,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972345873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:05,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:05,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:05,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:05,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38332 deadline: 1730972345873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:05,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:05,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:05,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:05,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:05,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:05,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972345983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:05,992 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:05,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:05,992 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:05,992 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:05,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38374 deadline: 1730972345983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:05,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:05,993 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:05,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972345983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:05,993 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:05,993 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:05,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:05,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:05,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:05,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:05,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972345992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:05,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:05,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:05,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:06,122 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/1d23c12a75334a289a845eead6958492 2024-11-07T09:38:06,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/bdf5d4e74f5f4b54afd978ad62194530 is 50, key is test_row_0/B:col10/1730972284690/Put/seqid=0 2024-11-07T09:38:06,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742293_1469 (size=12001) 2024-11-07T09:38:06,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:06,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972346203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:06,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:06,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38374 deadline: 1730972346203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:06,211 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:06,211 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:06,211 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:06,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:06,211 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:06,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972346203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:06,211 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:06,212 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:06,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:06,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:06,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:06,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:06,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972346203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:06,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:06,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:06,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:06,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:06,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972346523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:06,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:06,531 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:06,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38374 deadline: 1730972346523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:06,531 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:06,531 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:06,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:06,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972346523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:06,531 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:06,531 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:06,531 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:06,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:06,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:06,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:06,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:06,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972346523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:06,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:06,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:06,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:06,533 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/bdf5d4e74f5f4b54afd978ad62194530 2024-11-07T09:38:06,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/94a651b42b36474bb83956956acb1cfd is 50, key is test_row_0/C:col10/1730972284690/Put/seqid=0 2024-11-07T09:38:06,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742294_1470 (size=12001) 2024-11-07T09:38:06,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-07T09:38:06,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:06,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38332 deadline: 1730972346903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:06,912 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 9f1eb259a29b,42895,1730972153970 for 'test_row_1' in 57c7d380eebbfff219ec5446401d9b7d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T09:38:06,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:06,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:06,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:06,944 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/94a651b42b36474bb83956956acb1cfd 2024-11-07T09:38:06,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/1d23c12a75334a289a845eead6958492 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/1d23c12a75334a289a845eead6958492 2024-11-07T09:38:06,951 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/1d23c12a75334a289a845eead6958492, entries=150, sequenceid=37, filesize=11.7 K 2024-11-07T09:38:06,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/bdf5d4e74f5f4b54afd978ad62194530 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/bdf5d4e74f5f4b54afd978ad62194530 2024-11-07T09:38:06,954 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/bdf5d4e74f5f4b54afd978ad62194530, entries=150, sequenceid=37, filesize=11.7 K 2024-11-07T09:38:06,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/94a651b42b36474bb83956956acb1cfd as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/94a651b42b36474bb83956956acb1cfd 2024-11-07T09:38:06,958 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/94a651b42b36474bb83956956acb1cfd, entries=150, sequenceid=37, filesize=11.7 K 2024-11-07T09:38:06,958 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 57c7d380eebbfff219ec5446401d9b7d in 1245ms, sequenceid=37, compaction requested=false 2024-11-07T09:38:06,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 57c7d380eebbfff219ec5446401d9b7d: 2024-11-07T09:38:06,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:06,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-07T09:38:06,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-07T09:38:06,961 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-07T09:38:06,961 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3240 sec 2024-11-07T09:38:06,963 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 2.3310 sec 2024-11-07T09:38:07,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 57c7d380eebbfff219ec5446401d9b7d 2024-11-07T09:38:07,049 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:38:07,049 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 57c7d380eebbfff219ec5446401d9b7d 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-07T09:38:07,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/36c3401cdec544c88b50cccc13cdf582 is 50, key is test_row_0/A:col10/1730972287049/Put/seqid=0 2024-11-07T09:38:07,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742295_1471 (size=14341) 2024-11-07T09:38:07,070 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T09:38:07,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:07,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38374 deadline: 1730972347092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:07,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:07,103 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:07,103 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:07,103 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:07,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972347093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:07,104 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:07,104 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:07,104 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:07,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:07,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972347094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:07,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:07,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:07,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:07,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:07,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972347095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:07,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:07,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:07,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:07,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:07,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38374 deadline: 1730972347213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:07,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:07,222 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:07,222 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:07,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972347213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:07,222 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:07,223 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:07,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:07,223 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:07,223 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:07,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972347213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:07,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:07,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:07,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:07,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:07,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972347213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:07,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:07,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:07,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:07,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:07,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38374 deadline: 1730972347432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:07,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:07,443 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:07,443 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:07,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972347433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:07,443 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:07,444 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:07,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:07,444 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:07,444 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:07,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972347433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:07,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:07,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:07,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:07,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:07,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972347433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:07,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:07,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:07,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:07,466 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/36c3401cdec544c88b50cccc13cdf582 2024-11-07T09:38:07,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/c15b08bd21874ffd9048738c9782e66c is 50, key is test_row_0/B:col10/1730972287049/Put/seqid=0 2024-11-07T09:38:07,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742296_1472 (size=12001) 2024-11-07T09:38:07,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:07,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38374 deadline: 1730972347753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:07,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:07,767 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:07,767 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:07,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972347753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:07,767 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:07,767 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:07,767 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:07,767 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:07,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:07,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972347753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:07,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:07,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:07,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:07,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:07,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972347753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:07,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:07,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:07,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:07,877 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/c15b08bd21874ffd9048738c9782e66c 2024-11-07T09:38:07,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/67f23380cb5443e6ba28ebacb82f3865 is 50, key is test_row_0/C:col10/1730972287049/Put/seqid=0 2024-11-07T09:38:07,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742297_1473 (size=12001) 2024-11-07T09:38:08,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:08,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38374 deadline: 1730972348273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:08,282 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:08,282 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:08,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:08,282 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:08,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972348273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:08,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:08,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972348273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:08,282 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:08,282 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:08,282 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:08,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:08,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:08,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:08,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:08,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972348273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:08,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:08,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:08,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:08,287 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/67f23380cb5443e6ba28ebacb82f3865 2024-11-07T09:38:08,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/36c3401cdec544c88b50cccc13cdf582 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/36c3401cdec544c88b50cccc13cdf582 2024-11-07T09:38:08,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/36c3401cdec544c88b50cccc13cdf582, entries=200, sequenceid=53, filesize=14.0 K 2024-11-07T09:38:08,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/c15b08bd21874ffd9048738c9782e66c as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/c15b08bd21874ffd9048738c9782e66c 2024-11-07T09:38:08,298 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/c15b08bd21874ffd9048738c9782e66c, entries=150, sequenceid=53, filesize=11.7 K 2024-11-07T09:38:08,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/67f23380cb5443e6ba28ebacb82f3865 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/67f23380cb5443e6ba28ebacb82f3865 2024-11-07T09:38:08,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/67f23380cb5443e6ba28ebacb82f3865, entries=150, sequenceid=53, filesize=11.7 K 2024-11-07T09:38:08,302 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 57c7d380eebbfff219ec5446401d9b7d in 1253ms, sequenceid=53, compaction requested=true 2024-11-07T09:38:08,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 57c7d380eebbfff219ec5446401d9b7d: 2024-11-07T09:38:08,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c7d380eebbfff219ec5446401d9b7d:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T09:38:08,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:08,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c7d380eebbfff219ec5446401d9b7d:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T09:38:08,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:08,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c7d380eebbfff219ec5446401d9b7d:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:38:08,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:38:08,302 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:38:08,302 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:38:08,303 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43019 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:38:08,303 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:38:08,303 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 57c7d380eebbfff219ec5446401d9b7d/A is initiating minor compaction (all files) 2024-11-07T09:38:08,303 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 57c7d380eebbfff219ec5446401d9b7d/B is initiating minor compaction (all files) 2024-11-07T09:38:08,303 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57c7d380eebbfff219ec5446401d9b7d/A in TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:08,303 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57c7d380eebbfff219ec5446401d9b7d/B in TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:08,303 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/c9071ab0d6f44eb0a9904da1a3d163c4, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/1d23c12a75334a289a845eead6958492, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/36c3401cdec544c88b50cccc13cdf582] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp, totalSize=42.0 K 2024-11-07T09:38:08,303 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/5aa7244bae9c4cebac41dd6a47a1ac90, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/bdf5d4e74f5f4b54afd978ad62194530, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/c15b08bd21874ffd9048738c9782e66c] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp, totalSize=32.9 K 2024-11-07T09:38:08,304 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9071ab0d6f44eb0a9904da1a3d163c4, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1730972284644 2024-11-07T09:38:08,304 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 5aa7244bae9c4cebac41dd6a47a1ac90, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1730972284649 2024-11-07T09:38:08,304 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d23c12a75334a289a845eead6958492, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1730972284690 2024-11-07T09:38:08,304 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting bdf5d4e74f5f4b54afd978ad62194530, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1730972284690 2024-11-07T09:38:08,304 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36c3401cdec544c88b50cccc13cdf582, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1730972285870 2024-11-07T09:38:08,304 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting c15b08bd21874ffd9048738c9782e66c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1730972287048 2024-11-07T09:38:08,312 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57c7d380eebbfff219ec5446401d9b7d#A#compaction#401 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 65.38 MB/second 2024-11-07T09:38:08,312 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/5f31c2f4da0c4337a82a71179a52ebb5 is 50, key is test_row_0/A:col10/1730972287049/Put/seqid=0 2024-11-07T09:38:08,314 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57c7d380eebbfff219ec5446401d9b7d#B#compaction#402 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 65.38 MB/second 2024-11-07T09:38:08,314 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/37529645f7f04844a918c3794e5951fe is 50, key is test_row_0/B:col10/1730972287049/Put/seqid=0 2024-11-07T09:38:08,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742298_1474 (size=12104) 2024-11-07T09:38:08,327 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/5f31c2f4da0c4337a82a71179a52ebb5 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/5f31c2f4da0c4337a82a71179a52ebb5 2024-11-07T09:38:08,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742299_1475 (size=12104) 2024-11-07T09:38:08,331 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 57c7d380eebbfff219ec5446401d9b7d/A of 57c7d380eebbfff219ec5446401d9b7d into 5f31c2f4da0c4337a82a71179a52ebb5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:38:08,331 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57c7d380eebbfff219ec5446401d9b7d: 2024-11-07T09:38:08,331 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., storeName=57c7d380eebbfff219ec5446401d9b7d/A, priority=13, startTime=1730972288302; duration=0sec 2024-11-07T09:38:08,331 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:38:08,331 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c7d380eebbfff219ec5446401d9b7d:A 2024-11-07T09:38:08,331 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:38:08,332 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:38:08,332 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 57c7d380eebbfff219ec5446401d9b7d/C is initiating minor compaction (all files) 2024-11-07T09:38:08,332 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57c7d380eebbfff219ec5446401d9b7d/C in TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:08,332 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/76b11018c64e4d41a6fbe8c677fb9b56, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/94a651b42b36474bb83956956acb1cfd, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/67f23380cb5443e6ba28ebacb82f3865] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp, totalSize=32.9 K 2024-11-07T09:38:08,333 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76b11018c64e4d41a6fbe8c677fb9b56, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1730972284649 2024-11-07T09:38:08,333 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94a651b42b36474bb83956956acb1cfd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1730972284690 2024-11-07T09:38:08,333 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67f23380cb5443e6ba28ebacb82f3865, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1730972287048 2024-11-07T09:38:08,339 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57c7d380eebbfff219ec5446401d9b7d#C#compaction#403 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 65.38 MB/second 2024-11-07T09:38:08,339 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/3465c6bc8bae407ab85d499a5480b2b6 is 50, key is test_row_0/C:col10/1730972287049/Put/seqid=0 2024-11-07T09:38:08,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742300_1476 (size=12104) 2024-11-07T09:38:08,348 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/3465c6bc8bae407ab85d499a5480b2b6 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/3465c6bc8bae407ab85d499a5480b2b6 2024-11-07T09:38:08,353 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 57c7d380eebbfff219ec5446401d9b7d/C of 57c7d380eebbfff219ec5446401d9b7d into 3465c6bc8bae407ab85d499a5480b2b6(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:38:08,353 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57c7d380eebbfff219ec5446401d9b7d: 2024-11-07T09:38:08,353 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., storeName=57c7d380eebbfff219ec5446401d9b7d/C, priority=13, startTime=1730972288302; duration=0sec 2024-11-07T09:38:08,353 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:08,353 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c7d380eebbfff219ec5446401d9b7d:C 2024-11-07T09:38:08,731 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/37529645f7f04844a918c3794e5951fe as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/37529645f7f04844a918c3794e5951fe 2024-11-07T09:38:08,735 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 57c7d380eebbfff219ec5446401d9b7d/B of 57c7d380eebbfff219ec5446401d9b7d into 37529645f7f04844a918c3794e5951fe(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:38:08,735 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57c7d380eebbfff219ec5446401d9b7d: 2024-11-07T09:38:08,735 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., storeName=57c7d380eebbfff219ec5446401d9b7d/B, priority=13, startTime=1730972288302; duration=0sec 2024-11-07T09:38:08,735 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:08,735 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c7d380eebbfff219ec5446401d9b7d:B 2024-11-07T09:38:08,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-07T09:38:08,793 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:38:08,793 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T09:38:08,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-07T09:38:08,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-07T09:38:08,795 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T09:38:08,796 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T09:38:08,796 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T09:38:08,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-07T09:38:08,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 57c7d380eebbfff219ec5446401d9b7d 2024-11-07T09:38:08,924 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:38:08,924 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 57c7d380eebbfff219ec5446401d9b7d 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-07T09:38:08,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/b903f54c3043437fa9be0193adeb9633 is 50, key is test_row_0/A:col10/1730972287093/Put/seqid=0 2024-11-07T09:38:08,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742301_1477 (size=14341) 2024-11-07T09:38:08,933 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/b903f54c3043437fa9be0193adeb9633 2024-11-07T09:38:08,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/efacfcd7e7604169b927e3980b3fc2d3 is 50, key is test_row_0/B:col10/1730972287093/Put/seqid=0 2024-11-07T09:38:08,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742302_1478 (size=12001) 2024-11-07T09:38:08,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-07T09:38:08,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:08,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. as already flushing 2024-11-07T09:38:08,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:08,948 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:08,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:08,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:09,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:09,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38332 deadline: 1730972349003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:09,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:09,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:09,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:09,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-07T09:38:09,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:09,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. as already flushing 2024-11-07T09:38:09,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:09,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:09,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:09,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:09,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-07T09:38:09,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:09,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38332 deadline: 1730972349113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:09,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:09,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:09,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:09,254 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-07T09:38:09,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:09,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. as already flushing 2024-11-07T09:38:09,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:09,254 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:09,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:09,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:09,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:09,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38374 deadline: 1730972349293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:09,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972349293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:09,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972349293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:09,297 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 9f1eb259a29b,42895,1730972153970 for 'test_row_2' in 57c7d380eebbfff219ec5446401d9b7d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T09:38:09,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:09,297 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:09,297 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972349293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:09,297 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:09,297 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 9f1eb259a29b,42895,1730972153970 for 'test_row_2' in 57c7d380eebbfff219ec5446401d9b7d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T09:38:09,297 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:09,298 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:09,298 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:09,297 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 9f1eb259a29b,42895,1730972153970 for 'test_row_1' in 57c7d380eebbfff219ec5446401d9b7d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T09:38:09,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:09,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:09,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:09,298 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 9f1eb259a29b,42895,1730972153970 for 'test_row_2' in 57c7d380eebbfff219ec5446401d9b7d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T09:38:09,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:09,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:09,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:09,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:09,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38332 deadline: 1730972349333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:09,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:09,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:09,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:09,344 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/efacfcd7e7604169b927e3980b3fc2d3 2024-11-07T09:38:09,351 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/7edbf7b0310f4432a0f618509df25c17 is 50, key is test_row_0/C:col10/1730972287093/Put/seqid=0 2024-11-07T09:38:09,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742303_1479 (size=12001) 2024-11-07T09:38:09,355 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/7edbf7b0310f4432a0f618509df25c17 2024-11-07T09:38:09,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/b903f54c3043437fa9be0193adeb9633 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/b903f54c3043437fa9be0193adeb9633 2024-11-07T09:38:09,361 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/b903f54c3043437fa9be0193adeb9633, entries=200, sequenceid=77, filesize=14.0 K 2024-11-07T09:38:09,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/efacfcd7e7604169b927e3980b3fc2d3 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/efacfcd7e7604169b927e3980b3fc2d3 2024-11-07T09:38:09,365 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/efacfcd7e7604169b927e3980b3fc2d3, entries=150, sequenceid=77, filesize=11.7 K 2024-11-07T09:38:09,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/7edbf7b0310f4432a0f618509df25c17 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/7edbf7b0310f4432a0f618509df25c17 2024-11-07T09:38:09,368 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/7edbf7b0310f4432a0f618509df25c17, entries=150, sequenceid=77, filesize=11.7 K 2024-11-07T09:38:09,369 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 57c7d380eebbfff219ec5446401d9b7d in 445ms, sequenceid=77, compaction requested=false 2024-11-07T09:38:09,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 57c7d380eebbfff219ec5446401d9b7d: 2024-11-07T09:38:09,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-07T09:38:09,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:09,408 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 57c7d380eebbfff219ec5446401d9b7d 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-07T09:38:09,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/5c3445fd49d7439084dede48f0514d3a is 50, key is test_row_0/A:col10/1730972289002/Put/seqid=0 2024-11-07T09:38:09,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742304_1480 (size=12001) 2024-11-07T09:38:09,418 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/5c3445fd49d7439084dede48f0514d3a 2024-11-07T09:38:09,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-07T09:38:09,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/81ac7ef7c02949c1bb9892ef48979f5f is 50, key is test_row_0/B:col10/1730972289002/Put/seqid=0 2024-11-07T09:38:09,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742305_1481 (size=12001) 2024-11-07T09:38:09,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 57c7d380eebbfff219ec5446401d9b7d 2024-11-07T09:38:09,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. as already flushing 2024-11-07T09:38:09,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:09,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38332 deadline: 1730972349781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:09,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:09,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:09,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:09,829 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/81ac7ef7c02949c1bb9892ef48979f5f 2024-11-07T09:38:09,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/b3a9e8bd52f146039305d7fbe2576dd2 is 50, key is test_row_0/C:col10/1730972289002/Put/seqid=0 2024-11-07T09:38:09,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742306_1482 (size=12001) 2024-11-07T09:38:09,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:09,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38332 deadline: 1730972349893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:09,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:09,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:09,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:09,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-07T09:38:10,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:10,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38332 deadline: 1730972350103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:10,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:10,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:10,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:10,239 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/b3a9e8bd52f146039305d7fbe2576dd2 2024-11-07T09:38:10,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/5c3445fd49d7439084dede48f0514d3a as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/5c3445fd49d7439084dede48f0514d3a 2024-11-07T09:38:10,246 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/5c3445fd49d7439084dede48f0514d3a, entries=150, sequenceid=92, filesize=11.7 K 2024-11-07T09:38:10,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/81ac7ef7c02949c1bb9892ef48979f5f as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/81ac7ef7c02949c1bb9892ef48979f5f 2024-11-07T09:38:10,250 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/81ac7ef7c02949c1bb9892ef48979f5f, entries=150, sequenceid=92, filesize=11.7 K 2024-11-07T09:38:10,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/b3a9e8bd52f146039305d7fbe2576dd2 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/b3a9e8bd52f146039305d7fbe2576dd2 2024-11-07T09:38:10,253 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/b3a9e8bd52f146039305d7fbe2576dd2, entries=150, sequenceid=92, filesize=11.7 K 2024-11-07T09:38:10,254 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 57c7d380eebbfff219ec5446401d9b7d in 846ms, sequenceid=92, compaction requested=true 2024-11-07T09:38:10,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 57c7d380eebbfff219ec5446401d9b7d: 2024-11-07T09:38:10,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:10,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-07T09:38:10,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-07T09:38:10,257 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-07T09:38:10,257 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4600 sec 2024-11-07T09:38:10,259 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.4650 sec 2024-11-07T09:38:10,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 57c7d380eebbfff219ec5446401d9b7d 2024-11-07T09:38:10,429 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:38:10,429 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 57c7d380eebbfff219ec5446401d9b7d 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-07T09:38:10,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/c1d2a725fbbe4de991e24b0a940cfbe1 is 50, key is test_row_0/A:col10/1730972290428/Put/seqid=0 2024-11-07T09:38:10,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742307_1483 (size=14341) 2024-11-07T09:38:10,438 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/c1d2a725fbbe4de991e24b0a940cfbe1 2024-11-07T09:38:10,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/a78b7cac61dd4280bd24419246f5c7d6 is 50, key is test_row_0/B:col10/1730972290428/Put/seqid=0 2024-11-07T09:38:10,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742308_1484 (size=12001) 2024-11-07T09:38:10,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:10,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38332 deadline: 1730972350523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:10,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:10,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:10,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:10,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:10,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38332 deadline: 1730972350633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:10,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:10,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:10,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:10,848 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/a78b7cac61dd4280bd24419246f5c7d6 2024-11-07T09:38:10,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/3c5474de56634d2eaef2b4a63f9f7dd0 is 50, key is test_row_0/C:col10/1730972290428/Put/seqid=0 2024-11-07T09:38:10,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742309_1485 (size=12001) 2024-11-07T09:38:10,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:10,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38332 deadline: 1730972350853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:10,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:10,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:10,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:10,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-07T09:38:10,944 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:38:10,944 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T09:38:10,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-07T09:38:10,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-07T09:38:10,946 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T09:38:10,946 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T09:38:10,946 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T09:38:11,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-07T09:38:11,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-07T09:38:11,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:11,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. as already flushing 2024-11-07T09:38:11,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:11,098 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:11,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:11,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:11,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:11,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38332 deadline: 1730972351173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:11,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:11,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:11,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:11,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-07T09:38:11,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:11,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. as already flushing 2024-11-07T09:38:11,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:11,252 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:11,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:11,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:11,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/3c5474de56634d2eaef2b4a63f9f7dd0 2024-11-07T09:38:11,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/c1d2a725fbbe4de991e24b0a940cfbe1 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/c1d2a725fbbe4de991e24b0a940cfbe1 2024-11-07T09:38:11,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-07T09:38:11,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/c1d2a725fbbe4de991e24b0a940cfbe1, entries=200, sequenceid=114, filesize=14.0 K 2024-11-07T09:38:11,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/a78b7cac61dd4280bd24419246f5c7d6 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/a78b7cac61dd4280bd24419246f5c7d6 2024-11-07T09:38:11,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/a78b7cac61dd4280bd24419246f5c7d6, entries=150, sequenceid=114, filesize=11.7 K 2024-11-07T09:38:11,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/3c5474de56634d2eaef2b4a63f9f7dd0 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/3c5474de56634d2eaef2b4a63f9f7dd0 2024-11-07T09:38:11,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/3c5474de56634d2eaef2b4a63f9f7dd0, entries=150, sequenceid=114, filesize=11.7 K 2024-11-07T09:38:11,274 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 57c7d380eebbfff219ec5446401d9b7d in 845ms, sequenceid=114, compaction requested=true 2024-11-07T09:38:11,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 57c7d380eebbfff219ec5446401d9b7d: 2024-11-07T09:38:11,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c7d380eebbfff219ec5446401d9b7d:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T09:38:11,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:11,274 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T09:38:11,274 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T09:38:11,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c7d380eebbfff219ec5446401d9b7d:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T09:38:11,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:11,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c7d380eebbfff219ec5446401d9b7d:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:38:11,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:38:11,275 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52787 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T09:38:11,275 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 57c7d380eebbfff219ec5446401d9b7d/A is initiating minor compaction (all files) 2024-11-07T09:38:11,275 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57c7d380eebbfff219ec5446401d9b7d/A in TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:11,275 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/5f31c2f4da0c4337a82a71179a52ebb5, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/b903f54c3043437fa9be0193adeb9633, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/5c3445fd49d7439084dede48f0514d3a, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/c1d2a725fbbe4de991e24b0a940cfbe1] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp, totalSize=51.5 K 2024-11-07T09:38:11,275 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T09:38:11,275 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 57c7d380eebbfff219ec5446401d9b7d/B is initiating minor compaction (all files) 2024-11-07T09:38:11,275 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57c7d380eebbfff219ec5446401d9b7d/B in TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:11,275 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/37529645f7f04844a918c3794e5951fe, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/efacfcd7e7604169b927e3980b3fc2d3, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/81ac7ef7c02949c1bb9892ef48979f5f, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/a78b7cac61dd4280bd24419246f5c7d6] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp, totalSize=47.0 K 2024-11-07T09:38:11,276 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f31c2f4da0c4337a82a71179a52ebb5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1730972287048 2024-11-07T09:38:11,276 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 37529645f7f04844a918c3794e5951fe, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1730972287048 2024-11-07T09:38:11,276 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting b903f54c3043437fa9be0193adeb9633, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1730972287090 2024-11-07T09:38:11,276 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting efacfcd7e7604169b927e3980b3fc2d3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1730972287090 2024-11-07T09:38:11,276 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c3445fd49d7439084dede48f0514d3a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1730972288944 2024-11-07T09:38:11,276 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 81ac7ef7c02949c1bb9892ef48979f5f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1730972288944 2024-11-07T09:38:11,276 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1d2a725fbbe4de991e24b0a940cfbe1, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1730972289754 2024-11-07T09:38:11,277 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting a78b7cac61dd4280bd24419246f5c7d6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1730972289759 2024-11-07T09:38:11,284 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57c7d380eebbfff219ec5446401d9b7d#B#compaction#414 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 65.38 MB/second 2024-11-07T09:38:11,284 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/0a3609c4b4204a3cac30dcce2f63b859 is 50, key is test_row_0/B:col10/1730972290428/Put/seqid=0 2024-11-07T09:38:11,285 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57c7d380eebbfff219ec5446401d9b7d#A#compaction#413 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 65.38 MB/second 2024-11-07T09:38:11,285 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/299db1b4f43d45a5a70cc5a54d08fd53 is 50, key is test_row_0/A:col10/1730972290428/Put/seqid=0 2024-11-07T09:38:11,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742310_1486 (size=12241) 2024-11-07T09:38:11,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742311_1487 (size=12241) 2024-11-07T09:38:11,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 57c7d380eebbfff219ec5446401d9b7d 2024-11-07T09:38:11,320 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:38:11,320 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 57c7d380eebbfff219ec5446401d9b7d 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-07T09:38:11,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/dc8dbbfa484246bc9c69ab3beb84e5f4 is 50, key is test_row_0/A:col10/1730972290522/Put/seqid=0 2024-11-07T09:38:11,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742312_1488 (size=14391) 2024-11-07T09:38:11,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:11,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972351373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:11,387 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:11,387 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:11,387 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:11,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:11,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972351375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:11,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:11,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:11,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:11,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38374 deadline: 1730972351386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:11,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:11,391 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:11,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:11,391 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:11,391 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:11,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972351387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:11,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:11,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:11,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:11,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-07T09:38:11,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:11,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. as already flushing 2024-11-07T09:38:11,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:11,404 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:11,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:11,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:11,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:11,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972351492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:11,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:11,504 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:11,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972351493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:11,504 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:11,504 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:11,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:11,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972351493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:11,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:11,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:11,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:11,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:11,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:11,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:11,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38374 deadline: 1730972351493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:11,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:11,505 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:11,505 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:11,505 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:11,558 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-07T09:38:11,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:11,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. as already flushing 2024-11-07T09:38:11,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:11,558 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:11,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:11,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:11,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-07T09:38:11,695 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/0a3609c4b4204a3cac30dcce2f63b859 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/0a3609c4b4204a3cac30dcce2f63b859 2024-11-07T09:38:11,698 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/299db1b4f43d45a5a70cc5a54d08fd53 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/299db1b4f43d45a5a70cc5a54d08fd53 2024-11-07T09:38:11,700 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 57c7d380eebbfff219ec5446401d9b7d/B of 57c7d380eebbfff219ec5446401d9b7d into 0a3609c4b4204a3cac30dcce2f63b859(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:38:11,700 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57c7d380eebbfff219ec5446401d9b7d: 2024-11-07T09:38:11,700 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., storeName=57c7d380eebbfff219ec5446401d9b7d/B, priority=12, startTime=1730972291274; duration=0sec 2024-11-07T09:38:11,700 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:38:11,701 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c7d380eebbfff219ec5446401d9b7d:B 2024-11-07T09:38:11,701 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T09:38:11,702 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T09:38:11,702 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 57c7d380eebbfff219ec5446401d9b7d/C is initiating minor compaction (all files) 2024-11-07T09:38:11,702 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57c7d380eebbfff219ec5446401d9b7d/C in TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:11,702 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 57c7d380eebbfff219ec5446401d9b7d/A of 57c7d380eebbfff219ec5446401d9b7d into 299db1b4f43d45a5a70cc5a54d08fd53(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:38:11,702 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57c7d380eebbfff219ec5446401d9b7d: 2024-11-07T09:38:11,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:11,702 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/3465c6bc8bae407ab85d499a5480b2b6, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/7edbf7b0310f4432a0f618509df25c17, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/b3a9e8bd52f146039305d7fbe2576dd2, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/3c5474de56634d2eaef2b4a63f9f7dd0] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp, totalSize=47.0 K 2024-11-07T09:38:11,702 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., storeName=57c7d380eebbfff219ec5446401d9b7d/A, priority=12, startTime=1730972291274; duration=0sec 2024-11-07T09:38:11,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38332 deadline: 1730972351693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:11,702 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:11,702 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c7d380eebbfff219ec5446401d9b7d:A 2024-11-07T09:38:11,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:11,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:11,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:11,703 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 3465c6bc8bae407ab85d499a5480b2b6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1730972287048 2024-11-07T09:38:11,703 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 7edbf7b0310f4432a0f618509df25c17, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1730972287090 2024-11-07T09:38:11,704 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting b3a9e8bd52f146039305d7fbe2576dd2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1730972288944 2024-11-07T09:38:11,704 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c5474de56634d2eaef2b4a63f9f7dd0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1730972289759 2024-11-07T09:38:11,710 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57c7d380eebbfff219ec5446401d9b7d#C#compaction#416 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 65.38 MB/second 2024-11-07T09:38:11,710 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/1e7c5b716d26422e9d63af09ed84b7d0 is 50, key is test_row_0/C:col10/1730972290428/Put/seqid=0 2024-11-07T09:38:11,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-07T09:38:11,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:11,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. as already flushing 2024-11-07T09:38:11,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:11,711 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:11,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:11,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:11,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742313_1489 (size=12241) 2024-11-07T09:38:11,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:11,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972351713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:11,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:11,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:11,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:11,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38374 deadline: 1730972351714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:11,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:11,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:11,725 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:11,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972351714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:11,725 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:11,725 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:11,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:11,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972351714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:11,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:11,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:11,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:11,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:11,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:11,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:11,728 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/dc8dbbfa484246bc9c69ab3beb84e5f4 2024-11-07T09:38:11,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/532a56968230480db6dae542994fdbfe is 50, key is test_row_0/B:col10/1730972290522/Put/seqid=0 2024-11-07T09:38:11,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742314_1490 (size=12051) 2024-11-07T09:38:11,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-07T09:38:11,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:11,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. as already flushing 2024-11-07T09:38:11,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:11,864 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:11,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:11,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:12,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-07T09:38:12,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:12,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. as already flushing 2024-11-07T09:38:12,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:12,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:12,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:12,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:12,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:12,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972352033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:12,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:12,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:12,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38374 deadline: 1730972352033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:12,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:12,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:12,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:12,044 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:12,044 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:12,044 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:12,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972352033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:12,044 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:12,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:12,044 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:12,044 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:12,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972352033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:12,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:12,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:12,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:12,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-07T09:38:12,119 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/1e7c5b716d26422e9d63af09ed84b7d0 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/1e7c5b716d26422e9d63af09ed84b7d0 2024-11-07T09:38:12,122 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 57c7d380eebbfff219ec5446401d9b7d/C of 57c7d380eebbfff219ec5446401d9b7d into 1e7c5b716d26422e9d63af09ed84b7d0(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:38:12,122 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57c7d380eebbfff219ec5446401d9b7d: 2024-11-07T09:38:12,122 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., storeName=57c7d380eebbfff219ec5446401d9b7d/C, priority=12, startTime=1730972291274; duration=0sec 2024-11-07T09:38:12,123 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:12,123 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c7d380eebbfff219ec5446401d9b7d:C 2024-11-07T09:38:12,137 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/532a56968230480db6dae542994fdbfe 2024-11-07T09:38:12,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/a81ec63a2fad45518a89cd6241217ba1 is 50, key is test_row_0/C:col10/1730972290522/Put/seqid=0 2024-11-07T09:38:12,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742315_1491 (size=12051) 2024-11-07T09:38:12,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-07T09:38:12,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:12,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. as already flushing 2024-11-07T09:38:12,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:12,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:12,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:12,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:12,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-07T09:38:12,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:12,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. as already flushing 2024-11-07T09:38:12,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:12,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:12,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:12,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:12,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-07T09:38:12,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:12,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. as already flushing 2024-11-07T09:38:12,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:12,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:12,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:12,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:12,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/a81ec63a2fad45518a89cd6241217ba1 2024-11-07T09:38:12,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/dc8dbbfa484246bc9c69ab3beb84e5f4 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/dc8dbbfa484246bc9c69ab3beb84e5f4 2024-11-07T09:38:12,553 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/dc8dbbfa484246bc9c69ab3beb84e5f4, entries=200, sequenceid=129, filesize=14.1 K 2024-11-07T09:38:12,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/532a56968230480db6dae542994fdbfe as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/532a56968230480db6dae542994fdbfe 2024-11-07T09:38:12,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/532a56968230480db6dae542994fdbfe, entries=150, sequenceid=129, filesize=11.8 K 2024-11-07T09:38:12,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/a81ec63a2fad45518a89cd6241217ba1 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/a81ec63a2fad45518a89cd6241217ba1 2024-11-07T09:38:12,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/a81ec63a2fad45518a89cd6241217ba1, entries=150, sequenceid=129, filesize=11.8 K 2024-11-07T09:38:12,560 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 57c7d380eebbfff219ec5446401d9b7d in 1240ms, sequenceid=129, compaction requested=false 2024-11-07T09:38:12,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 57c7d380eebbfff219ec5446401d9b7d: 2024-11-07T09:38:12,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 57c7d380eebbfff219ec5446401d9b7d 2024-11-07T09:38:12,562 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:38:12,562 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 57c7d380eebbfff219ec5446401d9b7d 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-07T09:38:12,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/b04455418b3046ba90a54545f1957062 is 50, key is test_row_0/A:col10/1730972292561/Put/seqid=0 2024-11-07T09:38:12,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742316_1492 (size=16931) 2024-11-07T09:38:12,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:12,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972352579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:12,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:12,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38374 deadline: 1730972352580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:12,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:12,593 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:12,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:12,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:12,593 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:12,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:12,593 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:12,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972352591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:12,593 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:12,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:12,593 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:12,593 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:12,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972352591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:12,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:12,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:12,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:12,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-07T09:38:12,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:12,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. as already flushing 2024-11-07T09:38:12,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:12,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:12,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:12,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:12,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:12,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972352703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:12,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:12,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38374 deadline: 1730972352703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:12,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:12,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:12,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:12,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:12,711 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:12,711 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:12,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972352703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:12,711 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:12,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:12,711 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:12,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972352703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:12,711 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:12,711 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:12,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:12,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:12,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:12,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:12,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38332 deadline: 1730972352713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:12,725 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 9f1eb259a29b,42895,1730972153970 for 'test_row_0' in 57c7d380eebbfff219ec5446401d9b7d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T09:38:12,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:12,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:12,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:12,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-07T09:38:12,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:12,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. as already flushing 2024-11-07T09:38:12,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:12,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:12,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:12,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:12,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:12,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972352913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:12,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:12,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:12,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38374 deadline: 1730972352922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:12,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:12,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:12,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:12,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972352922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:12,925 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:12,925 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:12,925 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:12,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:12,925 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:12,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972352923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:12,925 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:12,925 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:12,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:12,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:12,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:12,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-07T09:38:12,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:12,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. as already flushing 2024-11-07T09:38:12,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:12,936 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:12,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:12,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:12,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/b04455418b3046ba90a54545f1957062 2024-11-07T09:38:12,980 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/aa9ca9750622482f8337923786a751d7 is 50, key is test_row_0/B:col10/1730972292561/Put/seqid=0 2024-11-07T09:38:12,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742317_1493 (size=12151) 2024-11-07T09:38:13,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-07T09:38:13,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:13,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. as already flushing 2024-11-07T09:38:13,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:13,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:13,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:13,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:13,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-07T09:38:13,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-07T09:38:13,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:13,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. as already flushing 2024-11-07T09:38:13,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:13,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:13,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:13,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972353233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:13,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:13,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:13,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:13,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:13,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972353233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:13,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:13,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:13,243 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:13,243 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:13,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38374 deadline: 1730972353233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:13,243 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:13,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:13,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:13,243 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:13,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972353233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:13,243 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:13,243 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:13,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:13,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:13,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:13,384 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/aa9ca9750622482f8337923786a751d7 2024-11-07T09:38:13,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/a9955098ac29427fa2a1917d7529fa0a is 50, key is test_row_0/C:col10/1730972292561/Put/seqid=0 2024-11-07T09:38:13,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742318_1494 (size=12151) 2024-11-07T09:38:13,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-07T09:38:13,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:13,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. as already flushing 2024-11-07T09:38:13,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:13,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:13,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:13,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:13,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-07T09:38:13,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:13,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. as already flushing 2024-11-07T09:38:13,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:13,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:13,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:13,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:13,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-07T09:38:13,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:13,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. as already flushing 2024-11-07T09:38:13,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:13,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:13,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:13,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:13,713 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-07T09:38:13,713 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-07T09:38:13,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:13,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972353753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:13,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:13,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38374 deadline: 1730972353753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:13,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:13,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:13,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:13,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972353753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:13,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:13,761 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:13,761 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:13,761 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:13,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:13,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972353753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:13,761 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:13,761 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:13,761 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:13,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:13,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:13,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:13,795 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/a9955098ac29427fa2a1917d7529fa0a 2024-11-07T09:38:13,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/b04455418b3046ba90a54545f1957062 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/b04455418b3046ba90a54545f1957062 2024-11-07T09:38:13,801 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/b04455418b3046ba90a54545f1957062, entries=250, sequenceid=156, filesize=16.5 K 2024-11-07T09:38:13,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/aa9ca9750622482f8337923786a751d7 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/aa9ca9750622482f8337923786a751d7 2024-11-07T09:38:13,805 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/aa9ca9750622482f8337923786a751d7, entries=150, sequenceid=156, filesize=11.9 K 2024-11-07T09:38:13,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/a9955098ac29427fa2a1917d7529fa0a as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/a9955098ac29427fa2a1917d7529fa0a 2024-11-07T09:38:13,808 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/a9955098ac29427fa2a1917d7529fa0a, entries=150, sequenceid=156, filesize=11.9 K 2024-11-07T09:38:13,809 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 57c7d380eebbfff219ec5446401d9b7d in 1247ms, sequenceid=156, compaction requested=true 2024-11-07T09:38:13,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 57c7d380eebbfff219ec5446401d9b7d: 2024-11-07T09:38:13,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c7d380eebbfff219ec5446401d9b7d:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T09:38:13,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:13,809 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:38:13,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c7d380eebbfff219ec5446401d9b7d:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T09:38:13,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:13,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c7d380eebbfff219ec5446401d9b7d:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:38:13,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:38:13,809 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:38:13,810 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43563 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:38:13,810 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 57c7d380eebbfff219ec5446401d9b7d/A is initiating minor compaction (all files) 2024-11-07T09:38:13,810 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57c7d380eebbfff219ec5446401d9b7d/A in TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:13,810 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:38:13,810 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 57c7d380eebbfff219ec5446401d9b7d/B is initiating minor compaction (all files) 2024-11-07T09:38:13,810 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/299db1b4f43d45a5a70cc5a54d08fd53, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/dc8dbbfa484246bc9c69ab3beb84e5f4, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/b04455418b3046ba90a54545f1957062] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp, totalSize=42.5 K 2024-11-07T09:38:13,810 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57c7d380eebbfff219ec5446401d9b7d/B in TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:13,810 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/0a3609c4b4204a3cac30dcce2f63b859, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/532a56968230480db6dae542994fdbfe, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/aa9ca9750622482f8337923786a751d7] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp, totalSize=35.6 K 2024-11-07T09:38:13,810 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 299db1b4f43d45a5a70cc5a54d08fd53, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1730972289759 2024-11-07T09:38:13,811 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a3609c4b4204a3cac30dcce2f63b859, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1730972289759 2024-11-07T09:38:13,811 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc8dbbfa484246bc9c69ab3beb84e5f4, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1730972290499 2024-11-07T09:38:13,811 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 532a56968230480db6dae542994fdbfe, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1730972290509 2024-11-07T09:38:13,812 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting b04455418b3046ba90a54545f1957062, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1730972291367 2024-11-07T09:38:13,812 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting aa9ca9750622482f8337923786a751d7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1730972291385 2024-11-07T09:38:13,819 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57c7d380eebbfff219ec5446401d9b7d#B#compaction#422 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 65.38 MB/second 2024-11-07T09:38:13,819 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/ac66b68b296b4a4fa3902a9a33401edf is 50, key is test_row_0/B:col10/1730972292561/Put/seqid=0 2024-11-07T09:38:13,820 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57c7d380eebbfff219ec5446401d9b7d#A#compaction#423 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 65.38 MB/second 2024-11-07T09:38:13,820 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/d04ed992507642d7a03d0ce61286c653 is 50, key is test_row_0/A:col10/1730972292561/Put/seqid=0 2024-11-07T09:38:13,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742319_1495 (size=12493) 2024-11-07T09:38:13,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742320_1496 (size=12493) 2024-11-07T09:38:13,829 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/d04ed992507642d7a03d0ce61286c653 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/d04ed992507642d7a03d0ce61286c653 2024-11-07T09:38:13,834 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 57c7d380eebbfff219ec5446401d9b7d/A of 57c7d380eebbfff219ec5446401d9b7d into d04ed992507642d7a03d0ce61286c653(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:38:13,834 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57c7d380eebbfff219ec5446401d9b7d: 2024-11-07T09:38:13,834 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., storeName=57c7d380eebbfff219ec5446401d9b7d/A, priority=13, startTime=1730972293809; duration=0sec 2024-11-07T09:38:13,834 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:38:13,834 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c7d380eebbfff219ec5446401d9b7d:A 2024-11-07T09:38:13,834 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:38:13,835 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:38:13,835 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 57c7d380eebbfff219ec5446401d9b7d/C is initiating minor compaction (all files) 2024-11-07T09:38:13,835 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57c7d380eebbfff219ec5446401d9b7d/C in TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:13,835 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/1e7c5b716d26422e9d63af09ed84b7d0, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/a81ec63a2fad45518a89cd6241217ba1, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/a9955098ac29427fa2a1917d7529fa0a] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp, totalSize=35.6 K 2024-11-07T09:38:13,835 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e7c5b716d26422e9d63af09ed84b7d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1730972289759 2024-11-07T09:38:13,835 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting a81ec63a2fad45518a89cd6241217ba1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1730972290509 2024-11-07T09:38:13,836 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9955098ac29427fa2a1917d7529fa0a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1730972291385 2024-11-07T09:38:13,842 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57c7d380eebbfff219ec5446401d9b7d#C#compaction#424 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 65.38 MB/second 2024-11-07T09:38:13,842 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/b7d2d5ed9c634995b0b9734abad22bbb is 50, key is test_row_0/C:col10/1730972292561/Put/seqid=0 2024-11-07T09:38:13,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742321_1497 (size=12493) 2024-11-07T09:38:13,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-07T09:38:13,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:13,854 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 57c7d380eebbfff219ec5446401d9b7d 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-07T09:38:13,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/557a5b5252ae4cd2b405dd9c745f3220 is 50, key is test_row_0/A:col10/1730972292575/Put/seqid=0 2024-11-07T09:38:13,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742322_1498 (size=12151) 2024-11-07T09:38:14,228 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/ac66b68b296b4a4fa3902a9a33401edf as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/ac66b68b296b4a4fa3902a9a33401edf 2024-11-07T09:38:14,232 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 57c7d380eebbfff219ec5446401d9b7d/B of 57c7d380eebbfff219ec5446401d9b7d into ac66b68b296b4a4fa3902a9a33401edf(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:38:14,232 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57c7d380eebbfff219ec5446401d9b7d: 2024-11-07T09:38:14,232 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., storeName=57c7d380eebbfff219ec5446401d9b7d/B, priority=13, startTime=1730972293809; duration=0sec 2024-11-07T09:38:14,232 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:14,232 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c7d380eebbfff219ec5446401d9b7d:B 2024-11-07T09:38:14,249 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/b7d2d5ed9c634995b0b9734abad22bbb as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/b7d2d5ed9c634995b0b9734abad22bbb 2024-11-07T09:38:14,253 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 57c7d380eebbfff219ec5446401d9b7d/C of 57c7d380eebbfff219ec5446401d9b7d into b7d2d5ed9c634995b0b9734abad22bbb(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:38:14,253 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57c7d380eebbfff219ec5446401d9b7d: 2024-11-07T09:38:14,253 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., storeName=57c7d380eebbfff219ec5446401d9b7d/C, priority=13, startTime=1730972293809; duration=0sec 2024-11-07T09:38:14,253 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:14,253 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c7d380eebbfff219ec5446401d9b7d:C 2024-11-07T09:38:14,263 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/557a5b5252ae4cd2b405dd9c745f3220 2024-11-07T09:38:14,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/007ec9fdae354c9da7dea77106181ae4 is 50, key is test_row_0/B:col10/1730972292575/Put/seqid=0 2024-11-07T09:38:14,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742323_1499 (size=12151) 2024-11-07T09:38:14,674 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/007ec9fdae354c9da7dea77106181ae4 2024-11-07T09:38:14,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/58fc255aa42a45f1beaef33cf7e37161 is 50, key is test_row_0/C:col10/1730972292575/Put/seqid=0 2024-11-07T09:38:14,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742324_1500 (size=12151) 2024-11-07T09:38:14,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 57c7d380eebbfff219ec5446401d9b7d 2024-11-07T09:38:14,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. as already flushing 2024-11-07T09:38:14,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:14,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38332 deadline: 1730972354843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:14,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:14,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:14,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:14,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:14,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972354843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:14,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:14,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:14,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:14,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:14,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38374 deadline: 1730972354843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:14,856 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:14,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:14,856 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:14,856 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:14,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972354844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:14,856 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:14,856 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:14,856 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:14,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:14,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972354845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:14,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:14,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:14,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:14,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:14,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38332 deadline: 1730972354963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:14,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:14,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:14,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:14,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972354963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:14,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:14,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:14,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:14,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:14,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38374 deadline: 1730972354963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:14,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:14,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:14,966 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:14,966 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:14,966 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:14,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972354963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:14,966 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:14,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:14,966 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:14,966 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:14,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972354964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:14,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:14,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:14,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:15,088 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/58fc255aa42a45f1beaef33cf7e37161 2024-11-07T09:38:15,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/557a5b5252ae4cd2b405dd9c745f3220 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/557a5b5252ae4cd2b405dd9c745f3220 2024-11-07T09:38:15,095 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/557a5b5252ae4cd2b405dd9c745f3220, entries=150, sequenceid=170, filesize=11.9 K 2024-11-07T09:38:15,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/007ec9fdae354c9da7dea77106181ae4 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/007ec9fdae354c9da7dea77106181ae4 2024-11-07T09:38:15,100 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/007ec9fdae354c9da7dea77106181ae4, entries=150, sequenceid=170, filesize=11.9 K 2024-11-07T09:38:15,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/58fc255aa42a45f1beaef33cf7e37161 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/58fc255aa42a45f1beaef33cf7e37161 2024-11-07T09:38:15,105 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/58fc255aa42a45f1beaef33cf7e37161, entries=150, sequenceid=170, filesize=11.9 K 2024-11-07T09:38:15,106 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 57c7d380eebbfff219ec5446401d9b7d in 1252ms, sequenceid=170, compaction requested=false 2024-11-07T09:38:15,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 57c7d380eebbfff219ec5446401d9b7d: 2024-11-07T09:38:15,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:15,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-07T09:38:15,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-07T09:38:15,109 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-07T09:38:15,109 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.1620 sec 2024-11-07T09:38:15,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-07T09:38:15,114 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 4.1680 sec 2024-11-07T09:38:15,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 57c7d380eebbfff219ec5446401d9b7d 2024-11-07T09:38:15,177 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:38:15,177 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 57c7d380eebbfff219ec5446401d9b7d 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T09:38:15,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/4e555b765d9b4e8e9493db6df0227273 is 50, key is test_row_0/A:col10/1730972294809/Put/seqid=0 2024-11-07T09:38:15,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742325_1501 (size=14541) 2024-11-07T09:38:15,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:15,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972355192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:15,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:15,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:15,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:15,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:15,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38332 deadline: 1730972355192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:15,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:15,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:15,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:15,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:15,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972355193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:15,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:15,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:15,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:15,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:15,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38374 deadline: 1730972355194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:15,208 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:15,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:15,208 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:15,208 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:15,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972355195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:15,208 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:15,208 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:15,208 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:15,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:15,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972355303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:15,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:15,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:15,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38332 deadline: 1730972355303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:15,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:15,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:15,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:15,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:15,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:15,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:15,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972355313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:15,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:15,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:15,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:15,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:15,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38374 deadline: 1730972355313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:15,324 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:15,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:15,324 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:15,324 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:15,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972355313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:15,324 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:15,324 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:15,324 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:15,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:15,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972355512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:15,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:15,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38332 deadline: 1730972355513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:15,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:15,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:15,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:15,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:15,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:15,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:15,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:15,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972355533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:15,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:15,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38374 deadline: 1730972355533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:15,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:15,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:15,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:15,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:15,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972355533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:15,546 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:15,546 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:15,546 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:15,546 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:15,546 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:15,546 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:15,588 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/4e555b765d9b4e8e9493db6df0227273 2024-11-07T09:38:15,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/1408287bb2eb4fc8b7b59cf835916f22 is 50, key is test_row_0/B:col10/1730972294809/Put/seqid=0 2024-11-07T09:38:15,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742326_1502 (size=12151) 2024-11-07T09:38:15,607 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/1408287bb2eb4fc8b7b59cf835916f22 2024-11-07T09:38:15,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/3aafd6af766f4879875a5cba8a64f224 is 50, key is test_row_0/C:col10/1730972294809/Put/seqid=0 2024-11-07T09:38:15,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742327_1503 (size=12151) 2024-11-07T09:38:15,629 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/3aafd6af766f4879875a5cba8a64f224 2024-11-07T09:38:15,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/4e555b765d9b4e8e9493db6df0227273 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/4e555b765d9b4e8e9493db6df0227273 2024-11-07T09:38:15,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/4e555b765d9b4e8e9493db6df0227273, entries=200, sequenceid=196, filesize=14.2 K 2024-11-07T09:38:15,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/1408287bb2eb4fc8b7b59cf835916f22 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/1408287bb2eb4fc8b7b59cf835916f22 2024-11-07T09:38:15,647 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/1408287bb2eb4fc8b7b59cf835916f22, entries=150, sequenceid=196, filesize=11.9 K 2024-11-07T09:38:15,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/3aafd6af766f4879875a5cba8a64f224 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/3aafd6af766f4879875a5cba8a64f224 2024-11-07T09:38:15,652 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/3aafd6af766f4879875a5cba8a64f224, entries=150, sequenceid=196, filesize=11.9 K 2024-11-07T09:38:15,653 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 57c7d380eebbfff219ec5446401d9b7d in 476ms, sequenceid=196, compaction requested=true 2024-11-07T09:38:15,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 57c7d380eebbfff219ec5446401d9b7d: 2024-11-07T09:38:15,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c7d380eebbfff219ec5446401d9b7d:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T09:38:15,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:15,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c7d380eebbfff219ec5446401d9b7d:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T09:38:15,653 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:38:15,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:15,653 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:38:15,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c7d380eebbfff219ec5446401d9b7d:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:38:15,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:38:15,654 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:38:15,654 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:38:15,654 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 57c7d380eebbfff219ec5446401d9b7d/B is initiating minor compaction (all files) 2024-11-07T09:38:15,654 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 57c7d380eebbfff219ec5446401d9b7d/A is initiating minor compaction (all files) 2024-11-07T09:38:15,654 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57c7d380eebbfff219ec5446401d9b7d/A in TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:15,654 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57c7d380eebbfff219ec5446401d9b7d/B in TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:15,654 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/d04ed992507642d7a03d0ce61286c653, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/557a5b5252ae4cd2b405dd9c745f3220, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/4e555b765d9b4e8e9493db6df0227273] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp, totalSize=38.3 K 2024-11-07T09:38:15,654 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/ac66b68b296b4a4fa3902a9a33401edf, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/007ec9fdae354c9da7dea77106181ae4, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/1408287bb2eb4fc8b7b59cf835916f22] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp, totalSize=35.9 K 2024-11-07T09:38:15,656 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting ac66b68b296b4a4fa3902a9a33401edf, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1730972291385 2024-11-07T09:38:15,656 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting d04ed992507642d7a03d0ce61286c653, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1730972291385 2024-11-07T09:38:15,656 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 007ec9fdae354c9da7dea77106181ae4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1730972292575 2024-11-07T09:38:15,656 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 557a5b5252ae4cd2b405dd9c745f3220, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1730972292575 2024-11-07T09:38:15,657 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 1408287bb2eb4fc8b7b59cf835916f22, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1730972294809 2024-11-07T09:38:15,657 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e555b765d9b4e8e9493db6df0227273, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1730972294799 2024-11-07T09:38:15,663 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57c7d380eebbfff219ec5446401d9b7d#A#compaction#431 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 65.38 MB/second 2024-11-07T09:38:15,664 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/2711b4ae4b504926af5ffc4b2d80a380 is 50, key is test_row_0/A:col10/1730972294809/Put/seqid=0 2024-11-07T09:38:15,669 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57c7d380eebbfff219ec5446401d9b7d#B#compaction#432 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 65.38 MB/second 2024-11-07T09:38:15,669 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/8fc3cdf2547a4a618108cc922c649073 is 50, key is test_row_0/B:col10/1730972294809/Put/seqid=0 2024-11-07T09:38:15,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742328_1504 (size=12595) 2024-11-07T09:38:15,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742329_1505 (size=12595) 2024-11-07T09:38:15,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 57c7d380eebbfff219ec5446401d9b7d 2024-11-07T09:38:15,827 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:38:15,827 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 57c7d380eebbfff219ec5446401d9b7d 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-07T09:38:15,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/34590c4672184e3198e8b8938a98746e is 50, key is test_row_0/A:col10/1730972295181/Put/seqid=0 2024-11-07T09:38:15,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742330_1506 (size=14541) 2024-11-07T09:38:15,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/34590c4672184e3198e8b8938a98746e 2024-11-07T09:38:15,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/836ba69fe46149429e5b0552a696b8b6 is 50, key is test_row_0/B:col10/1730972295181/Put/seqid=0 2024-11-07T09:38:15,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742331_1507 (size=12151) 2024-11-07T09:38:15,850 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/836ba69fe46149429e5b0552a696b8b6 2024-11-07T09:38:15,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/854b9b595c0c417fbe03f4ee42bec207 is 50, key is test_row_0/C:col10/1730972295181/Put/seqid=0 2024-11-07T09:38:15,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742332_1508 (size=12151) 2024-11-07T09:38:15,862 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/854b9b595c0c417fbe03f4ee42bec207 2024-11-07T09:38:15,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/34590c4672184e3198e8b8938a98746e as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/34590c4672184e3198e8b8938a98746e 2024-11-07T09:38:15,871 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/34590c4672184e3198e8b8938a98746e, entries=200, sequenceid=209, filesize=14.2 K 2024-11-07T09:38:15,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/836ba69fe46149429e5b0552a696b8b6 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/836ba69fe46149429e5b0552a696b8b6 2024-11-07T09:38:15,875 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/836ba69fe46149429e5b0552a696b8b6, entries=150, sequenceid=209, filesize=11.9 K 2024-11-07T09:38:15,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/854b9b595c0c417fbe03f4ee42bec207 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/854b9b595c0c417fbe03f4ee42bec207 2024-11-07T09:38:15,879 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/854b9b595c0c417fbe03f4ee42bec207, entries=150, sequenceid=209, filesize=11.9 K 2024-11-07T09:38:15,879 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=73.80 KB/75570 for 57c7d380eebbfff219ec5446401d9b7d in 52ms, sequenceid=209, compaction requested=true 2024-11-07T09:38:15,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 57c7d380eebbfff219ec5446401d9b7d: 2024-11-07T09:38:15,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c7d380eebbfff219ec5446401d9b7d:A, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:38:15,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T09:38:15,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c7d380eebbfff219ec5446401d9b7d:B, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:38:15,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-07T09:38:15,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c7d380eebbfff219ec5446401d9b7d:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:38:15,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-07T09:38:15,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 57c7d380eebbfff219ec5446401d9b7d 2024-11-07T09:38:15,881 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:38:15,881 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 57c7d380eebbfff219ec5446401d9b7d 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-07T09:38:15,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/1ce0912af74a4ecaa3749088986cefe0 is 50, key is test_row_0/A:col10/1730972295869/Put/seqid=0 2024-11-07T09:38:15,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742333_1509 (size=14541) 2024-11-07T09:38:15,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:15,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38374 deadline: 1730972355919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:15,936 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:15,936 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:15,936 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:15,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:15,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972355931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:15,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:15,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:15,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:15,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:15,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38332 deadline: 1730972355933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:15,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:15,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:15,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:15,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:15,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972355934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:15,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:15,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972355935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:15,953 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:15,953 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:15,953 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:15,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:15,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:15,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:16,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:16,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38374 deadline: 1730972356043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:16,050 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:16,050 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:16,050 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:16,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:16,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972356052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:16,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:16,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:16,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:16,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:16,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38332 deadline: 1730972356053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:16,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:16,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:16,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:16,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:16,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972356063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:16,077 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:16,077 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:16,077 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:16,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:16,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972356063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:16,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:16,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:16,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:16,083 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/2711b4ae4b504926af5ffc4b2d80a380 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/2711b4ae4b504926af5ffc4b2d80a380 2024-11-07T09:38:16,090 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 57c7d380eebbfff219ec5446401d9b7d/A of 57c7d380eebbfff219ec5446401d9b7d into 2711b4ae4b504926af5ffc4b2d80a380(size=12.3 K), total size for store is 26.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:38:16,090 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57c7d380eebbfff219ec5446401d9b7d: 2024-11-07T09:38:16,090 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., storeName=57c7d380eebbfff219ec5446401d9b7d/A, priority=13, startTime=1730972295653; duration=0sec 2024-11-07T09:38:16,090 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-07T09:38:16,090 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c7d380eebbfff219ec5446401d9b7d:A 2024-11-07T09:38:16,090 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c7d380eebbfff219ec5446401d9b7d:A 2024-11-07T09:38:16,090 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T09:38:16,096 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T09:38:16,097 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 57c7d380eebbfff219ec5446401d9b7d/C is initiating minor compaction (all files) 2024-11-07T09:38:16,097 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57c7d380eebbfff219ec5446401d9b7d/C in TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:16,097 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/b7d2d5ed9c634995b0b9734abad22bbb, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/58fc255aa42a45f1beaef33cf7e37161, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/3aafd6af766f4879875a5cba8a64f224, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/854b9b595c0c417fbe03f4ee42bec207] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp, totalSize=47.8 K 2024-11-07T09:38:16,097 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7d2d5ed9c634995b0b9734abad22bbb, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1730972291385 2024-11-07T09:38:16,097 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58fc255aa42a45f1beaef33cf7e37161, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1730972292575 2024-11-07T09:38:16,098 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3aafd6af766f4879875a5cba8a64f224, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1730972294809 2024-11-07T09:38:16,098 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 854b9b595c0c417fbe03f4ee42bec207, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1730972295181 2024-11-07T09:38:16,099 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/8fc3cdf2547a4a618108cc922c649073 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/8fc3cdf2547a4a618108cc922c649073 2024-11-07T09:38:16,107 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 57c7d380eebbfff219ec5446401d9b7d/B of 57c7d380eebbfff219ec5446401d9b7d into 8fc3cdf2547a4a618108cc922c649073(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:38:16,107 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57c7d380eebbfff219ec5446401d9b7d: 2024-11-07T09:38:16,107 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., storeName=57c7d380eebbfff219ec5446401d9b7d/B, priority=13, startTime=1730972295653; duration=0sec 2024-11-07T09:38:16,107 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-07T09:38:16,107 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c7d380eebbfff219ec5446401d9b7d:B 2024-11-07T09:38:16,107 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c7d380eebbfff219ec5446401d9b7d:B 2024-11-07T09:38:16,107 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 4 compacting, 0 eligible, 16 blocking 2024-11-07T09:38:16,108 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-07T09:38:16,108 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-07T09:38:16,108 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. because compaction request was cancelled 2024-11-07T09:38:16,108 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c7d380eebbfff219ec5446401d9b7d:C 2024-11-07T09:38:16,108 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-07T09:38:16,109 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57c7d380eebbfff219ec5446401d9b7d#C#compaction#437 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 65.38 MB/second 2024-11-07T09:38:16,109 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/e863f47d2fe6449cba3f48a66ecb4524 is 50, key is test_row_0/C:col10/1730972295181/Put/seqid=0 2024-11-07T09:38:16,110 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-07T09:38:16,110 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-07T09:38:16,110 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. because compaction request was cancelled 2024-11-07T09:38:16,110 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c7d380eebbfff219ec5446401d9b7d:B 2024-11-07T09:38:16,110 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-07T09:38:16,111 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-07T09:38:16,111 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-07T09:38:16,111 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. because compaction request was cancelled 2024-11-07T09:38:16,111 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c7d380eebbfff219ec5446401d9b7d:A 2024-11-07T09:38:16,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742334_1510 (size=12629) 2024-11-07T09:38:16,122 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/e863f47d2fe6449cba3f48a66ecb4524 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/e863f47d2fe6449cba3f48a66ecb4524 2024-11-07T09:38:16,126 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 57c7d380eebbfff219ec5446401d9b7d/C of 57c7d380eebbfff219ec5446401d9b7d into e863f47d2fe6449cba3f48a66ecb4524(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:38:16,126 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57c7d380eebbfff219ec5446401d9b7d: 2024-11-07T09:38:16,126 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., storeName=57c7d380eebbfff219ec5446401d9b7d/C, priority=12, startTime=1730972295880; duration=0sec 2024-11-07T09:38:16,126 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:16,126 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c7d380eebbfff219ec5446401d9b7d:C 2024-11-07T09:38:16,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:16,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38374 deadline: 1730972356253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:16,266 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:16,266 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:16,267 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:16,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:16,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972356273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:16,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:16,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38332 deadline: 1730972356273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:16,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:16,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:16,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:16,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:16,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:16,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:16,289 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/1ce0912af74a4ecaa3749088986cefe0 2024-11-07T09:38:16,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:16,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972356283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:16,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:16,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:16,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:16,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:16,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972356283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:16,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:16,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:16,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:16,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/6f573031dc2c4e08a3ca4c499dd076a0 is 50, key is test_row_0/B:col10/1730972295869/Put/seqid=0 2024-11-07T09:38:16,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742335_1511 (size=12151) 2024-11-07T09:38:16,315 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/6f573031dc2c4e08a3ca4c499dd076a0 2024-11-07T09:38:16,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/3d0220c4d9e14c9c97e01b683b7af954 is 50, key is test_row_0/C:col10/1730972295869/Put/seqid=0 2024-11-07T09:38:16,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742336_1512 (size=12151) 2024-11-07T09:38:16,342 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/3d0220c4d9e14c9c97e01b683b7af954 2024-11-07T09:38:16,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/1ce0912af74a4ecaa3749088986cefe0 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/1ce0912af74a4ecaa3749088986cefe0 2024-11-07T09:38:16,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/1ce0912af74a4ecaa3749088986cefe0, entries=200, sequenceid=224, filesize=14.2 K 2024-11-07T09:38:16,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/6f573031dc2c4e08a3ca4c499dd076a0 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/6f573031dc2c4e08a3ca4c499dd076a0 2024-11-07T09:38:16,355 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/6f573031dc2c4e08a3ca4c499dd076a0, entries=150, sequenceid=224, filesize=11.9 K 2024-11-07T09:38:16,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/3d0220c4d9e14c9c97e01b683b7af954 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/3d0220c4d9e14c9c97e01b683b7af954 2024-11-07T09:38:16,359 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/3d0220c4d9e14c9c97e01b683b7af954, entries=150, sequenceid=224, filesize=11.9 K 2024-11-07T09:38:16,359 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 57c7d380eebbfff219ec5446401d9b7d in 478ms, sequenceid=224, compaction requested=true 2024-11-07T09:38:16,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 57c7d380eebbfff219ec5446401d9b7d: 2024-11-07T09:38:16,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c7d380eebbfff219ec5446401d9b7d:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T09:38:16,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:16,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c7d380eebbfff219ec5446401d9b7d:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T09:38:16,360 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:38:16,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:16,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c7d380eebbfff219ec5446401d9b7d:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:38:16,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:38:16,360 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:38:16,361 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:38:16,361 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 57c7d380eebbfff219ec5446401d9b7d/A is initiating minor compaction (all files) 2024-11-07T09:38:16,361 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57c7d380eebbfff219ec5446401d9b7d/A in TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:16,361 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/2711b4ae4b504926af5ffc4b2d80a380, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/34590c4672184e3198e8b8938a98746e, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/1ce0912af74a4ecaa3749088986cefe0] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp, totalSize=40.7 K 2024-11-07T09:38:16,362 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:38:16,362 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 57c7d380eebbfff219ec5446401d9b7d/B is initiating minor compaction (all files) 2024-11-07T09:38:16,362 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57c7d380eebbfff219ec5446401d9b7d/B in TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:16,362 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2711b4ae4b504926af5ffc4b2d80a380, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1730972294809 2024-11-07T09:38:16,363 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/8fc3cdf2547a4a618108cc922c649073, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/836ba69fe46149429e5b0552a696b8b6, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/6f573031dc2c4e08a3ca4c499dd076a0] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp, totalSize=36.0 K 2024-11-07T09:38:16,364 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34590c4672184e3198e8b8938a98746e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1730972295181 2024-11-07T09:38:16,364 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fc3cdf2547a4a618108cc922c649073, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1730972294809 2024-11-07T09:38:16,364 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ce0912af74a4ecaa3749088986cefe0, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1730972295847 2024-11-07T09:38:16,364 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 836ba69fe46149429e5b0552a696b8b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1730972295181 2024-11-07T09:38:16,365 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f573031dc2c4e08a3ca4c499dd076a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1730972295867 2024-11-07T09:38:16,374 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57c7d380eebbfff219ec5446401d9b7d#A#compaction#440 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 65.38 MB/second 2024-11-07T09:38:16,375 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/27f9b044fb2c4102b18b29c10eea7307 is 50, key is test_row_0/A:col10/1730972295869/Put/seqid=0 2024-11-07T09:38:16,378 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57c7d380eebbfff219ec5446401d9b7d#B#compaction#441 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 65.38 MB/second 2024-11-07T09:38:16,378 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/bd8d07ec0bfe46daa642847d5bfd468a is 50, key is test_row_0/B:col10/1730972295869/Put/seqid=0 2024-11-07T09:38:16,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742337_1513 (size=12697) 2024-11-07T09:38:16,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742338_1514 (size=12697) 2024-11-07T09:38:16,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 57c7d380eebbfff219ec5446401d9b7d 2024-11-07T09:38:16,578 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:38:16,579 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 57c7d380eebbfff219ec5446401d9b7d 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-07T09:38:16,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/82dd2474e43c4d12828d111c8bff84c0 is 50, key is test_row_0/A:col10/1730972295932/Put/seqid=0 2024-11-07T09:38:16,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742339_1515 (size=14541) 2024-11-07T09:38:16,604 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/82dd2474e43c4d12828d111c8bff84c0 2024-11-07T09:38:16,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/edf2de82d12f4a87af6725193630baa8 is 50, key is test_row_0/B:col10/1730972295932/Put/seqid=0 2024-11-07T09:38:16,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742340_1516 (size=12151) 2024-11-07T09:38:16,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:16,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972356619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:16,633 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:16,633 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:16,633 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:16,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:16,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972356619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:16,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:16,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:16,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:16,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:16,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972356619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:16,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:16,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:16,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:16,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:16,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38332 deadline: 1730972356620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:16,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:16,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:16,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:16,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:16,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38374 deadline: 1730972356632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:16,635 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:16,635 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:16,635 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:16,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:16,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972356743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:16,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:16,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972356743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:16,747 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:16,747 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:16,748 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:16,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:16,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:16,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:16,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:16,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972356744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:16,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:16,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38332 deadline: 1730972356744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:16,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:16,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:16,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:16,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:16,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:16,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:16,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:16,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38374 deadline: 1730972356744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:16,749 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:16,749 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:16,749 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:16,796 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/27f9b044fb2c4102b18b29c10eea7307 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/27f9b044fb2c4102b18b29c10eea7307 2024-11-07T09:38:16,800 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 57c7d380eebbfff219ec5446401d9b7d/A of 57c7d380eebbfff219ec5446401d9b7d into 27f9b044fb2c4102b18b29c10eea7307(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:38:16,800 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57c7d380eebbfff219ec5446401d9b7d: 2024-11-07T09:38:16,800 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., storeName=57c7d380eebbfff219ec5446401d9b7d/A, priority=13, startTime=1730972296360; duration=0sec 2024-11-07T09:38:16,800 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:38:16,800 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c7d380eebbfff219ec5446401d9b7d:A 2024-11-07T09:38:16,800 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-07T09:38:16,805 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-07T09:38:16,805 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-07T09:38:16,805 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. because compaction request was cancelled 2024-11-07T09:38:16,805 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c7d380eebbfff219ec5446401d9b7d:C 2024-11-07T09:38:16,807 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/bd8d07ec0bfe46daa642847d5bfd468a as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/bd8d07ec0bfe46daa642847d5bfd468a 2024-11-07T09:38:16,811 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 57c7d380eebbfff219ec5446401d9b7d/B of 57c7d380eebbfff219ec5446401d9b7d into bd8d07ec0bfe46daa642847d5bfd468a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:38:16,811 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57c7d380eebbfff219ec5446401d9b7d: 2024-11-07T09:38:16,811 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., storeName=57c7d380eebbfff219ec5446401d9b7d/B, priority=13, startTime=1730972296360; duration=0sec 2024-11-07T09:38:16,811 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:16,811 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c7d380eebbfff219ec5446401d9b7d:B 2024-11-07T09:38:16,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:16,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972356953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:16,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:16,963 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:16,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972356953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:16,963 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:16,963 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:16,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:16,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:16,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:16,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972356953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:16,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:16,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:16,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:16,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:16,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:16,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38332 deadline: 1730972356954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:16,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:16,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38374 deadline: 1730972356954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:16,965 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:16,965 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:16,965 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:16,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:16,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:16,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:17,023 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/edf2de82d12f4a87af6725193630baa8 2024-11-07T09:38:17,032 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/8fbb5bddeaa543dd9f68f9c51ed16c4b is 50, key is test_row_0/C:col10/1730972295932/Put/seqid=0 2024-11-07T09:38:17,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742341_1517 (size=12151) 2024-11-07T09:38:17,036 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/8fbb5bddeaa543dd9f68f9c51ed16c4b 2024-11-07T09:38:17,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/82dd2474e43c4d12828d111c8bff84c0 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/82dd2474e43c4d12828d111c8bff84c0 2024-11-07T09:38:17,045 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/82dd2474e43c4d12828d111c8bff84c0, entries=200, sequenceid=249, filesize=14.2 K 2024-11-07T09:38:17,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/edf2de82d12f4a87af6725193630baa8 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/edf2de82d12f4a87af6725193630baa8 2024-11-07T09:38:17,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/edf2de82d12f4a87af6725193630baa8, entries=150, sequenceid=249, filesize=11.9 K 2024-11-07T09:38:17,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/8fbb5bddeaa543dd9f68f9c51ed16c4b as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/8fbb5bddeaa543dd9f68f9c51ed16c4b 2024-11-07T09:38:17,053 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/8fbb5bddeaa543dd9f68f9c51ed16c4b, entries=150, sequenceid=249, filesize=11.9 K 2024-11-07T09:38:17,054 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 57c7d380eebbfff219ec5446401d9b7d in 475ms, sequenceid=249, compaction requested=true 2024-11-07T09:38:17,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 57c7d380eebbfff219ec5446401d9b7d: 2024-11-07T09:38:17,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c7d380eebbfff219ec5446401d9b7d:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T09:38:17,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:17,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c7d380eebbfff219ec5446401d9b7d:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T09:38:17,054 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-07T09:38:17,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:17,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c7d380eebbfff219ec5446401d9b7d:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:38:17,055 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-07T09:38:17,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:38:17,055 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-07T09:38:17,055 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-07T09:38:17,055 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. because compaction request was cancelled 2024-11-07T09:38:17,055 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c7d380eebbfff219ec5446401d9b7d:A 2024-11-07T09:38:17,056 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:38:17,056 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-07T09:38:17,056 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-07T09:38:17,057 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. because compaction request was cancelled 2024-11-07T09:38:17,057 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c7d380eebbfff219ec5446401d9b7d:B 2024-11-07T09:38:17,057 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:38:17,057 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 57c7d380eebbfff219ec5446401d9b7d/C is initiating minor compaction (all files) 2024-11-07T09:38:17,057 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57c7d380eebbfff219ec5446401d9b7d/C in TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:17,057 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/e863f47d2fe6449cba3f48a66ecb4524, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/3d0220c4d9e14c9c97e01b683b7af954, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/8fbb5bddeaa543dd9f68f9c51ed16c4b] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp, totalSize=36.1 K 2024-11-07T09:38:17,057 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting e863f47d2fe6449cba3f48a66ecb4524, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1730972295181 2024-11-07T09:38:17,059 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d0220c4d9e14c9c97e01b683b7af954, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1730972295867 2024-11-07T09:38:17,060 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8fbb5bddeaa543dd9f68f9c51ed16c4b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1730972295929 2024-11-07T09:38:17,067 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57c7d380eebbfff219ec5446401d9b7d#C#compaction#445 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 65.38 MB/second 2024-11-07T09:38:17,068 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/4907f3e5fd7d4921bccfdc2cf3e1a78b is 50, key is test_row_0/C:col10/1730972295932/Put/seqid=0 2024-11-07T09:38:17,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742342_1518 (size=12731) 2024-11-07T09:38:17,086 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/4907f3e5fd7d4921bccfdc2cf3e1a78b as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/4907f3e5fd7d4921bccfdc2cf3e1a78b 2024-11-07T09:38:17,091 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 57c7d380eebbfff219ec5446401d9b7d/C of 57c7d380eebbfff219ec5446401d9b7d into 4907f3e5fd7d4921bccfdc2cf3e1a78b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:38:17,091 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57c7d380eebbfff219ec5446401d9b7d: 2024-11-07T09:38:17,091 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., storeName=57c7d380eebbfff219ec5446401d9b7d/C, priority=13, startTime=1730972297055; duration=0sec 2024-11-07T09:38:17,091 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:17,091 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c7d380eebbfff219ec5446401d9b7d:C 2024-11-07T09:38:17,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 57c7d380eebbfff219ec5446401d9b7d 2024-11-07T09:38:17,279 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:38:17,279 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 57c7d380eebbfff219ec5446401d9b7d 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-07T09:38:17,284 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/3281bf9b98e8493bbc72919faf3ed1c4 is 50, key is test_row_0/A:col10/1730972296616/Put/seqid=0 2024-11-07T09:38:17,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742343_1519 (size=14741) 2024-11-07T09:38:17,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:17,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972357314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:17,327 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:17,327 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:17,327 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:17,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:17,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972357320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:17,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:17,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38332 deadline: 1730972357321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:17,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:17,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:17,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:17,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:17,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:17,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:17,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:17,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972357327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:17,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:17,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38374 deadline: 1730972357327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:17,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:17,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:17,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:17,341 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:17,341 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:17,341 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:17,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:17,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972357433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:17,444 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:17,444 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:17,444 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:17,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:17,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972357444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:17,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:17,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:17,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38332 deadline: 1730972357444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:17,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:17,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:17,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:17,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:17,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:17,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38374 deadline: 1730972357444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:17,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:17,449 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:17,449 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:17,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:17,449 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:17,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972357444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:17,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:17,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:17,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:17,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:17,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972357653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:17,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:17,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:17,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:17,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:17,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972357654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:17,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:17,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:17,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:17,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:17,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972357654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:17,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:17,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:17,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:17,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:17,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38374 deadline: 1730972357654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:17,665 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:17,665 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:17,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:17,666 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:17,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38332 deadline: 1730972357654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:17,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:17,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:17,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:17,688 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/3281bf9b98e8493bbc72919faf3ed1c4 2024-11-07T09:38:17,697 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/6e78229fb200475694b74ddf6bddb954 is 50, key is test_row_0/B:col10/1730972296616/Put/seqid=0 2024-11-07T09:38:17,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742344_1520 (size=12301) 2024-11-07T09:38:17,703 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/6e78229fb200475694b74ddf6bddb954 2024-11-07T09:38:17,713 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/598b178fdb5a42b188d1c9737bca1d44 is 50, key is test_row_0/C:col10/1730972296616/Put/seqid=0 2024-11-07T09:38:17,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742345_1521 (size=12301) 2024-11-07T09:38:17,717 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/598b178fdb5a42b188d1c9737bca1d44 2024-11-07T09:38:17,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/3281bf9b98e8493bbc72919faf3ed1c4 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/3281bf9b98e8493bbc72919faf3ed1c4 2024-11-07T09:38:17,725 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/3281bf9b98e8493bbc72919faf3ed1c4, entries=200, sequenceid=267, filesize=14.4 K 2024-11-07T09:38:17,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/6e78229fb200475694b74ddf6bddb954 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/6e78229fb200475694b74ddf6bddb954 2024-11-07T09:38:17,729 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/6e78229fb200475694b74ddf6bddb954, entries=150, sequenceid=267, filesize=12.0 K 2024-11-07T09:38:17,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/598b178fdb5a42b188d1c9737bca1d44 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/598b178fdb5a42b188d1c9737bca1d44 2024-11-07T09:38:17,743 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/598b178fdb5a42b188d1c9737bca1d44, entries=150, sequenceid=267, filesize=12.0 K 2024-11-07T09:38:17,744 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 57c7d380eebbfff219ec5446401d9b7d in 465ms, sequenceid=267, compaction requested=true 2024-11-07T09:38:17,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 57c7d380eebbfff219ec5446401d9b7d: 2024-11-07T09:38:17,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c7d380eebbfff219ec5446401d9b7d:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T09:38:17,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:17,744 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:38:17,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c7d380eebbfff219ec5446401d9b7d:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T09:38:17,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:17,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c7d380eebbfff219ec5446401d9b7d:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:38:17,745 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:38:17,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:38:17,746 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41979 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:38:17,746 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:38:17,746 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 57c7d380eebbfff219ec5446401d9b7d/B is initiating minor compaction (all files) 2024-11-07T09:38:17,746 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 57c7d380eebbfff219ec5446401d9b7d/A is initiating minor compaction (all files) 2024-11-07T09:38:17,746 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57c7d380eebbfff219ec5446401d9b7d/B in TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:17,746 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57c7d380eebbfff219ec5446401d9b7d/A in TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:17,746 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/bd8d07ec0bfe46daa642847d5bfd468a, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/edf2de82d12f4a87af6725193630baa8, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/6e78229fb200475694b74ddf6bddb954] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp, totalSize=36.3 K 2024-11-07T09:38:17,746 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/27f9b044fb2c4102b18b29c10eea7307, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/82dd2474e43c4d12828d111c8bff84c0, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/3281bf9b98e8493bbc72919faf3ed1c4] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp, totalSize=41.0 K 2024-11-07T09:38:17,748 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting bd8d07ec0bfe46daa642847d5bfd468a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1730972295867 2024-11-07T09:38:17,748 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27f9b044fb2c4102b18b29c10eea7307, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1730972295867 2024-11-07T09:38:17,749 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting edf2de82d12f4a87af6725193630baa8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1730972295929 2024-11-07T09:38:17,749 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82dd2474e43c4d12828d111c8bff84c0, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1730972295929 2024-11-07T09:38:17,749 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e78229fb200475694b74ddf6bddb954, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1730972296616 2024-11-07T09:38:17,749 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3281bf9b98e8493bbc72919faf3ed1c4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1730972296616 2024-11-07T09:38:17,757 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57c7d380eebbfff219ec5446401d9b7d#B#compaction#449 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 65.38 MB/second 2024-11-07T09:38:17,757 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/f788b378995249e68b70889f0e9fad59 is 50, key is test_row_0/B:col10/1730972296616/Put/seqid=0 2024-11-07T09:38:17,760 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57c7d380eebbfff219ec5446401d9b7d#A#compaction#450 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 65.38 MB/second 2024-11-07T09:38:17,760 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/0d2fb9f52fd84848b3581cdf352bc997 is 50, key is test_row_0/A:col10/1730972296616/Put/seqid=0 2024-11-07T09:38:17,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742347_1523 (size=12949) 2024-11-07T09:38:17,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742346_1522 (size=12949) 2024-11-07T09:38:17,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 57c7d380eebbfff219ec5446401d9b7d 2024-11-07T09:38:17,979 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:38:17,980 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 57c7d380eebbfff219ec5446401d9b7d 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-07T09:38:17,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/026f839b06f54dfdb157a97853b80221 is 50, key is test_row_0/A:col10/1730972297296/Put/seqid=0 2024-11-07T09:38:17,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742348_1524 (size=12301) 2024-11-07T09:38:18,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:18,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972357990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:18,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:18,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:18,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:18,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:18,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972357991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:18,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:18,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:18,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:18,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:18,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38332 deadline: 1730972357994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:18,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:18,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:18,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:18,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:18,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38374 deadline: 1730972357996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:18,008 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:18,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:18,008 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:18,008 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:18,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972357997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:18,008 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:18,008 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:18,008 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:18,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:18,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972358113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:18,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:18,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:18,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:18,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:18,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972358113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:18,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:18,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:18,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:18,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:18,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38332 deadline: 1730972358113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:18,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:18,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:18,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:18,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:18,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38374 deadline: 1730972358113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:18,121 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:18,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:18,121 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:18,121 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:18,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972358113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:18,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:18,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:18,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:18,169 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/f788b378995249e68b70889f0e9fad59 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/f788b378995249e68b70889f0e9fad59 2024-11-07T09:38:18,169 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/0d2fb9f52fd84848b3581cdf352bc997 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/0d2fb9f52fd84848b3581cdf352bc997 2024-11-07T09:38:18,174 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 57c7d380eebbfff219ec5446401d9b7d/A of 57c7d380eebbfff219ec5446401d9b7d into 0d2fb9f52fd84848b3581cdf352bc997(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:38:18,174 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 57c7d380eebbfff219ec5446401d9b7d/B of 57c7d380eebbfff219ec5446401d9b7d into f788b378995249e68b70889f0e9fad59(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:38:18,174 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57c7d380eebbfff219ec5446401d9b7d: 2024-11-07T09:38:18,174 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57c7d380eebbfff219ec5446401d9b7d: 2024-11-07T09:38:18,174 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., storeName=57c7d380eebbfff219ec5446401d9b7d/A, priority=13, startTime=1730972297744; duration=0sec 2024-11-07T09:38:18,174 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., storeName=57c7d380eebbfff219ec5446401d9b7d/B, priority=13, startTime=1730972297744; duration=0sec 2024-11-07T09:38:18,174 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:38:18,174 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c7d380eebbfff219ec5446401d9b7d:A 2024-11-07T09:38:18,174 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:18,174 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c7d380eebbfff219ec5446401d9b7d:B 2024-11-07T09:38:18,174 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-07T09:38:18,175 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-07T09:38:18,175 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-07T09:38:18,175 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. because compaction request was cancelled 2024-11-07T09:38:18,175 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c7d380eebbfff219ec5446401d9b7d:C 2024-11-07T09:38:18,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:18,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972358323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:18,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:18,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:18,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:18,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972358323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:18,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:18,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:18,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:18,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:18,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:18,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38332 deadline: 1730972358323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:18,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:18,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:18,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:18,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:18,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38374 deadline: 1730972358333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:18,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:18,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972358333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:18,343 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:18,343 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:18,343 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:18,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:18,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:18,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:18,389 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/026f839b06f54dfdb157a97853b80221 2024-11-07T09:38:18,396 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/e249c89857fe4158899d8368be79153c is 50, key is test_row_0/B:col10/1730972297296/Put/seqid=0 2024-11-07T09:38:18,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742349_1525 (size=12301) 2024-11-07T09:38:18,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:18,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972358633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:18,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:18,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:18,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:18,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:18,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972358642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:18,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:18,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:18,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38332 deadline: 1730972358643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:18,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:18,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:18,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:18,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:18,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:18,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:18,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38374 deadline: 1730972358653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:18,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:18,662 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:18,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972358653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:18,662 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:18,662 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:18,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:18,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:18,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:18,800 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/e249c89857fe4158899d8368be79153c 2024-11-07T09:38:18,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/bdb9358f627944929075729b7b07f865 is 50, key is test_row_0/C:col10/1730972297296/Put/seqid=0 2024-11-07T09:38:18,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742350_1526 (size=12301) 2024-11-07T09:38:19,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-07T09:38:19,133 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:38:19,133 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T09:38:19,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-07T09:38:19,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-07T09:38:19,135 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T09:38:19,136 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T09:38:19,136 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T09:38:19,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:19,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972359153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:19,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:19,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:19,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:19,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:19,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972359163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:19,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:19,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:19,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:19,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:19,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38332 deadline: 1730972359163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:19,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:19,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:19,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:19,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:19,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38374 deadline: 1730972359173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:19,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:19,183 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:19,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972359173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:19,183 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:19,183 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:19,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:19,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:19,184 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:19,211 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/bdb9358f627944929075729b7b07f865 2024-11-07T09:38:19,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/026f839b06f54dfdb157a97853b80221 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/026f839b06f54dfdb157a97853b80221 2024-11-07T09:38:19,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/026f839b06f54dfdb157a97853b80221, entries=150, sequenceid=290, filesize=12.0 K 2024-11-07T09:38:19,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/e249c89857fe4158899d8368be79153c as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/e249c89857fe4158899d8368be79153c 2024-11-07T09:38:19,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/e249c89857fe4158899d8368be79153c, entries=150, sequenceid=290, filesize=12.0 K 2024-11-07T09:38:19,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/bdb9358f627944929075729b7b07f865 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/bdb9358f627944929075729b7b07f865 2024-11-07T09:38:19,225 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/bdb9358f627944929075729b7b07f865, entries=150, sequenceid=290, filesize=12.0 K 2024-11-07T09:38:19,226 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 57c7d380eebbfff219ec5446401d9b7d in 1247ms, sequenceid=290, compaction requested=true 2024-11-07T09:38:19,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 57c7d380eebbfff219ec5446401d9b7d: 2024-11-07T09:38:19,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c7d380eebbfff219ec5446401d9b7d:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T09:38:19,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:19,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c7d380eebbfff219ec5446401d9b7d:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T09:38:19,226 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-07T09:38:19,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:19,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c7d380eebbfff219ec5446401d9b7d:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:38:19,226 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-07T09:38:19,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:38:19,227 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-07T09:38:19,227 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-07T09:38:19,227 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-07T09:38:19,227 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-07T09:38:19,227 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. because compaction request was cancelled 2024-11-07T09:38:19,227 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. because compaction request was cancelled 2024-11-07T09:38:19,227 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c7d380eebbfff219ec5446401d9b7d:B 2024-11-07T09:38:19,227 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c7d380eebbfff219ec5446401d9b7d:A 2024-11-07T09:38:19,227 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:38:19,227 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:38:19,228 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 57c7d380eebbfff219ec5446401d9b7d/C is initiating minor compaction (all files) 2024-11-07T09:38:19,228 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57c7d380eebbfff219ec5446401d9b7d/C in TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:19,228 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/4907f3e5fd7d4921bccfdc2cf3e1a78b, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/598b178fdb5a42b188d1c9737bca1d44, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/bdb9358f627944929075729b7b07f865] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp, totalSize=36.5 K 2024-11-07T09:38:19,228 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 4907f3e5fd7d4921bccfdc2cf3e1a78b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1730972295929 2024-11-07T09:38:19,228 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 598b178fdb5a42b188d1c9737bca1d44, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1730972296616 2024-11-07T09:38:19,228 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting bdb9358f627944929075729b7b07f865, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1730972297296 2024-11-07T09:38:19,235 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57c7d380eebbfff219ec5446401d9b7d#C#compaction#454 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 65.38 MB/second 2024-11-07T09:38:19,236 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/eb0fa217ebb248dd8fe1d0b42dc570c7 is 50, key is test_row_0/C:col10/1730972297296/Put/seqid=0 2024-11-07T09:38:19,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742351_1527 (size=12983) 2024-11-07T09:38:19,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-07T09:38:19,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-07T09:38:19,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:19,288 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing 57c7d380eebbfff219ec5446401d9b7d 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-07T09:38:19,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/08949a90ae85487fb167a877e552494c is 50, key is test_row_0/A:col10/1730972297993/Put/seqid=0 2024-11-07T09:38:19,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742352_1528 (size=12301) 2024-11-07T09:38:19,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-07T09:38:19,645 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/eb0fa217ebb248dd8fe1d0b42dc570c7 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/eb0fa217ebb248dd8fe1d0b42dc570c7 2024-11-07T09:38:19,648 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 57c7d380eebbfff219ec5446401d9b7d/C of 57c7d380eebbfff219ec5446401d9b7d into eb0fa217ebb248dd8fe1d0b42dc570c7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:38:19,648 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57c7d380eebbfff219ec5446401d9b7d: 2024-11-07T09:38:19,649 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., storeName=57c7d380eebbfff219ec5446401d9b7d/C, priority=13, startTime=1730972299226; duration=0sec 2024-11-07T09:38:19,649 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:19,649 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c7d380eebbfff219ec5446401d9b7d:C 2024-11-07T09:38:19,696 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/08949a90ae85487fb167a877e552494c 2024-11-07T09:38:19,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/da1851007fe047898657afd2b6a2df04 is 50, key is test_row_0/B:col10/1730972297993/Put/seqid=0 2024-11-07T09:38:19,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742353_1529 (size=12301) 2024-11-07T09:38:19,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-07T09:38:20,109 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/da1851007fe047898657afd2b6a2df04 2024-11-07T09:38:20,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/4288bf1548444502bdbf819c863dbeb4 is 50, key is test_row_0/C:col10/1730972297993/Put/seqid=0 2024-11-07T09:38:20,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742354_1530 (size=12301) 2024-11-07T09:38:20,120 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/4288bf1548444502bdbf819c863dbeb4 2024-11-07T09:38:20,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/08949a90ae85487fb167a877e552494c as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/08949a90ae85487fb167a877e552494c 2024-11-07T09:38:20,135 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/08949a90ae85487fb167a877e552494c, entries=150, sequenceid=305, filesize=12.0 K 2024-11-07T09:38:20,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/da1851007fe047898657afd2b6a2df04 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/da1851007fe047898657afd2b6a2df04 2024-11-07T09:38:20,140 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/da1851007fe047898657afd2b6a2df04, entries=150, sequenceid=305, filesize=12.0 K 2024-11-07T09:38:20,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/4288bf1548444502bdbf819c863dbeb4 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/4288bf1548444502bdbf819c863dbeb4 2024-11-07T09:38:20,147 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/4288bf1548444502bdbf819c863dbeb4, entries=150, sequenceid=305, filesize=12.0 K 2024-11-07T09:38:20,149 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=0 B/0 for 57c7d380eebbfff219ec5446401d9b7d in 861ms, sequenceid=305, compaction requested=true 2024-11-07T09:38:20,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for 57c7d380eebbfff219ec5446401d9b7d: 2024-11-07T09:38:20,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:20,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-07T09:38:20,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-07T09:38:20,153 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-07T09:38:20,153 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0150 sec 2024-11-07T09:38:20,155 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 1.0210 sec 2024-11-07T09:38:20,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 57c7d380eebbfff219ec5446401d9b7d 2024-11-07T09:38:20,189 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:38:20,190 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 57c7d380eebbfff219ec5446401d9b7d 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T09:38:20,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/8ac30e0476c2438480538d5dec9a7536 is 50, key is test_row_0/A:col10/1730972300186/Put/seqid=0 2024-11-07T09:38:20,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742355_1531 (size=14741) 2024-11-07T09:38:20,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:20,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972360236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:20,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:20,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38374 deadline: 1730972360245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:20,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:20,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:20,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:20,250 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:20,250 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:20,250 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:20,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:20,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972360247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:20,260 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:20,260 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:20,260 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:20,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:20,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972360248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:20,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:20,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:20,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:20,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:20,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38332 deadline: 1730972360249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:20,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:20,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:20,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:20,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-07T09:38:20,273 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:38:20,274 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T09:38:20,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-07T09:38:20,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-07T09:38:20,275 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T09:38:20,276 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T09:38:20,276 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T09:38:20,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:20,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972360353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:20,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:20,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:20,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38374 deadline: 1730972360353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:20,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:20,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:20,362 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:20,362 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:20,362 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:20,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:20,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972360363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:20,376 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:20,376 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:20,376 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:20,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-07T09:38:20,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:20,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972360373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:20,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:20,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38332 deadline: 1730972360373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:20,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:20,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:20,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:20,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:20,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:20,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:20,428 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-07T09:38:20,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:20,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. as already flushing 2024-11-07T09:38:20,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:20,429 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:20,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:20,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:20,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-07T09:38:20,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:20,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. as already flushing 2024-11-07T09:38:20,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:20,582 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:20,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:20,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:20,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:20,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972360573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:20,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:20,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:20,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:20,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:20,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38374 deadline: 1730972360573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:20,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:20,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:20,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:20,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:20,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972360584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:20,587 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:20,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:20,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:20,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-07T09:38:20,599 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/8ac30e0476c2438480538d5dec9a7536 2024-11-07T09:38:20,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:20,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972360593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:20,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:20,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:20,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:20,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38332 deadline: 1730972360593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:20,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:20,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:20,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:20,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:20,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/c3756d6adea94a7fbecb14da86a08e3c is 50, key is test_row_0/B:col10/1730972300186/Put/seqid=0 2024-11-07T09:38:20,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742356_1532 (size=12301) 2024-11-07T09:38:20,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-07T09:38:20,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:20,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. as already flushing 2024-11-07T09:38:20,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:20,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:20,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:20,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:20,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-07T09:38:20,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:20,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. as already flushing 2024-11-07T09:38:20,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:20,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:20,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:20,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:20,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:20,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972360893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:20,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:20,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:20,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-07T09:38:20,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972360893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:20,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:20,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:20,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:20,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:20,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38374 deadline: 1730972360893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:20,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:20,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:20,903 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:20,903 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:20,903 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:20,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:20,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972360913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:20,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:20,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:20,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:20,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38332 deadline: 1730972360913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:20,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:20,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:20,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:20,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:21,009 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/c3756d6adea94a7fbecb14da86a08e3c 2024-11-07T09:38:21,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/e5cc5bc122cf4b61af1ddb5ab0f90935 is 50, key is test_row_0/C:col10/1730972300186/Put/seqid=0 2024-11-07T09:38:21,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742357_1533 (size=12301) 2024-11-07T09:38:21,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-07T09:38:21,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:21,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. as already flushing 2024-11-07T09:38:21,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:21,041 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:21,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:21,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:21,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-07T09:38:21,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:21,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. as already flushing 2024-11-07T09:38:21,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:21,194 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:21,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:21,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:21,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-07T09:38:21,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:21,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. as already flushing 2024-11-07T09:38:21,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:21,347 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:21,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:21,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:21,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-07T09:38:21,420 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/e5cc5bc122cf4b61af1ddb5ab0f90935 2024-11-07T09:38:21,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:21,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972361412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:21,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:21,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:21,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:21,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972361413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:21,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:21,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:21,422 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:21,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38374 deadline: 1730972361413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:21,422 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:21,422 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:21,422 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:21,422 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:21,422 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:21,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/8ac30e0476c2438480538d5dec9a7536 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/8ac30e0476c2438480538d5dec9a7536 2024-11-07T09:38:21,427 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/8ac30e0476c2438480538d5dec9a7536, entries=200, sequenceid=317, filesize=14.4 K 2024-11-07T09:38:21,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/c3756d6adea94a7fbecb14da86a08e3c as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/c3756d6adea94a7fbecb14da86a08e3c 2024-11-07T09:38:21,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/c3756d6adea94a7fbecb14da86a08e3c, entries=150, sequenceid=317, filesize=12.0 K 2024-11-07T09:38:21,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/e5cc5bc122cf4b61af1ddb5ab0f90935 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/e5cc5bc122cf4b61af1ddb5ab0f90935 2024-11-07T09:38:21,434 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/e5cc5bc122cf4b61af1ddb5ab0f90935, entries=150, sequenceid=317, filesize=12.0 K 2024-11-07T09:38:21,435 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 57c7d380eebbfff219ec5446401d9b7d in 1245ms, sequenceid=317, compaction requested=true 2024-11-07T09:38:21,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 57c7d380eebbfff219ec5446401d9b7d: 2024-11-07T09:38:21,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c7d380eebbfff219ec5446401d9b7d:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T09:38:21,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:21,435 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T09:38:21,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c7d380eebbfff219ec5446401d9b7d:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T09:38:21,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:21,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c7d380eebbfff219ec5446401d9b7d:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:38:21,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:38:21,435 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T09:38:21,438 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T09:38:21,438 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52292 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T09:38:21,438 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 57c7d380eebbfff219ec5446401d9b7d/B is initiating minor compaction (all files) 2024-11-07T09:38:21,438 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 57c7d380eebbfff219ec5446401d9b7d/A is initiating minor compaction (all files) 2024-11-07T09:38:21,439 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57c7d380eebbfff219ec5446401d9b7d/B in TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:21,439 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57c7d380eebbfff219ec5446401d9b7d/A in TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:21,439 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/f788b378995249e68b70889f0e9fad59, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/e249c89857fe4158899d8368be79153c, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/da1851007fe047898657afd2b6a2df04, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/c3756d6adea94a7fbecb14da86a08e3c] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp, totalSize=48.7 K 2024-11-07T09:38:21,439 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/0d2fb9f52fd84848b3581cdf352bc997, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/026f839b06f54dfdb157a97853b80221, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/08949a90ae85487fb167a877e552494c, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/8ac30e0476c2438480538d5dec9a7536] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp, totalSize=51.1 K 2024-11-07T09:38:21,439 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting f788b378995249e68b70889f0e9fad59, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1730972296616 2024-11-07T09:38:21,439 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d2fb9f52fd84848b3581cdf352bc997, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1730972296616 2024-11-07T09:38:21,439 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 026f839b06f54dfdb157a97853b80221, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1730972297296 2024-11-07T09:38:21,439 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting e249c89857fe4158899d8368be79153c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1730972297296 2024-11-07T09:38:21,439 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting da1851007fe047898657afd2b6a2df04, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1730972297993 2024-11-07T09:38:21,439 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08949a90ae85487fb167a877e552494c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1730972297993 2024-11-07T09:38:21,440 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting c3756d6adea94a7fbecb14da86a08e3c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1730972300186 2024-11-07T09:38:21,440 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ac30e0476c2438480538d5dec9a7536, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1730972300179 2024-11-07T09:38:21,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 57c7d380eebbfff219ec5446401d9b7d 2024-11-07T09:38:21,442 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:38:21,442 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 57c7d380eebbfff219ec5446401d9b7d 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-07T09:38:21,449 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57c7d380eebbfff219ec5446401d9b7d#B#compaction#461 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 65.38 MB/second 2024-11-07T09:38:21,449 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57c7d380eebbfff219ec5446401d9b7d#A#compaction#462 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 65.38 MB/second 2024-11-07T09:38:21,449 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/acc61ee4c222475bae3be822f39ae7b1 is 50, key is test_row_0/A:col10/1730972300186/Put/seqid=0 2024-11-07T09:38:21,449 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/0b3b50f45f2b442abba24afc59a90e4d is 50, key is test_row_0/B:col10/1730972300186/Put/seqid=0 2024-11-07T09:38:21,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/72a25f251703445f9078eb746f3a31c6 is 50, key is test_row_0/A:col10/1730972300235/Put/seqid=0 2024-11-07T09:38:21,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742359_1535 (size=13085) 2024-11-07T09:38:21,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742358_1534 (size=13085) 2024-11-07T09:38:21,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742360_1536 (size=14741) 2024-11-07T09:38:21,467 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/72a25f251703445f9078eb746f3a31c6 2024-11-07T09:38:21,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/322749f809d04d14a4dfd5f9b534de35 is 50, key is test_row_0/B:col10/1730972300235/Put/seqid=0 2024-11-07T09:38:21,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742361_1537 (size=12301) 2024-11-07T09:38:21,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:21,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972361468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:21,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:21,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:21,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:21,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:21,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38332 deadline: 1730972361472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:21,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:21,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:21,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:21,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-07T09:38:21,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:21,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. as already flushing 2024-11-07T09:38:21,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:21,500 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:21,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:21,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:21,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:21,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972361583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:21,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:21,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:21,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:21,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:21,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38332 deadline: 1730972361593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:21,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:21,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:21,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:21,653 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-07T09:38:21,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:21,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. as already flushing 2024-11-07T09:38:21,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:21,653 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:21,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:21,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:21,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-07T09:38:21,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:21,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. as already flushing 2024-11-07T09:38:21,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:21,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:21,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:21,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:21,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:21,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972361802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:21,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:21,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:21,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:21,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:21,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38332 deadline: 1730972361812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:21,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:21,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:21,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:21,864 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/0b3b50f45f2b442abba24afc59a90e4d as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/0b3b50f45f2b442abba24afc59a90e4d 2024-11-07T09:38:21,866 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/acc61ee4c222475bae3be822f39ae7b1 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/acc61ee4c222475bae3be822f39ae7b1 2024-11-07T09:38:21,868 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 57c7d380eebbfff219ec5446401d9b7d/B of 57c7d380eebbfff219ec5446401d9b7d into 0b3b50f45f2b442abba24afc59a90e4d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:38:21,869 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57c7d380eebbfff219ec5446401d9b7d: 2024-11-07T09:38:21,869 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., storeName=57c7d380eebbfff219ec5446401d9b7d/B, priority=12, startTime=1730972301435; duration=0sec 2024-11-07T09:38:21,869 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:38:21,869 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c7d380eebbfff219ec5446401d9b7d:B 2024-11-07T09:38:21,869 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:38:21,870 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:38:21,870 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 57c7d380eebbfff219ec5446401d9b7d/C is initiating minor compaction (all files) 2024-11-07T09:38:21,870 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57c7d380eebbfff219ec5446401d9b7d/C in TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:21,870 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/eb0fa217ebb248dd8fe1d0b42dc570c7, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/4288bf1548444502bdbf819c863dbeb4, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/e5cc5bc122cf4b61af1ddb5ab0f90935] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp, totalSize=36.7 K 2024-11-07T09:38:21,870 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting eb0fa217ebb248dd8fe1d0b42dc570c7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1730972297296 2024-11-07T09:38:21,870 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 4288bf1548444502bdbf819c863dbeb4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1730972297993 2024-11-07T09:38:21,871 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 57c7d380eebbfff219ec5446401d9b7d/A of 57c7d380eebbfff219ec5446401d9b7d into acc61ee4c222475bae3be822f39ae7b1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:38:21,871 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57c7d380eebbfff219ec5446401d9b7d: 2024-11-07T09:38:21,871 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., storeName=57c7d380eebbfff219ec5446401d9b7d/A, priority=12, startTime=1730972301435; duration=0sec 2024-11-07T09:38:21,871 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting e5cc5bc122cf4b61af1ddb5ab0f90935, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1730972300186 2024-11-07T09:38:21,871 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:21,871 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c7d380eebbfff219ec5446401d9b7d:A 2024-11-07T09:38:21,876 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/322749f809d04d14a4dfd5f9b534de35 2024-11-07T09:38:21,876 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57c7d380eebbfff219ec5446401d9b7d#C#compaction#465 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 65.38 MB/second 2024-11-07T09:38:21,877 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/77ae518b884f48758e72dcf042fdc58e is 50, key is test_row_0/C:col10/1730972300186/Put/seqid=0 2024-11-07T09:38:21,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742362_1538 (size=13085) 2024-11-07T09:38:21,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/7fe48d89569d409388c6fb766cb3ca6b is 50, key is test_row_0/C:col10/1730972300235/Put/seqid=0 2024-11-07T09:38:21,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742363_1539 (size=12301) 2024-11-07T09:38:21,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-07T09:38:21,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:21,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. as already flushing 2024-11-07T09:38:21,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:21,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:21,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:21,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:22,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-07T09:38:22,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:22,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. as already flushing 2024-11-07T09:38:22,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:22,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:22,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:22,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:22,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:22,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972362113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:22,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:22,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:22,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:22,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:22,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38332 deadline: 1730972362133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:22,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:22,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:22,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:22,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-07T09:38:22,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:22,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. as already flushing 2024-11-07T09:38:22,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:22,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:22,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:22,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:22,290 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/77ae518b884f48758e72dcf042fdc58e as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/77ae518b884f48758e72dcf042fdc58e 2024-11-07T09:38:22,291 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/7fe48d89569d409388c6fb766cb3ca6b 2024-11-07T09:38:22,294 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 57c7d380eebbfff219ec5446401d9b7d/C of 57c7d380eebbfff219ec5446401d9b7d into 77ae518b884f48758e72dcf042fdc58e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:38:22,294 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57c7d380eebbfff219ec5446401d9b7d: 2024-11-07T09:38:22,294 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., storeName=57c7d380eebbfff219ec5446401d9b7d/C, priority=13, startTime=1730972301435; duration=0sec 2024-11-07T09:38:22,294 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:22,294 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c7d380eebbfff219ec5446401d9b7d:C 2024-11-07T09:38:22,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/72a25f251703445f9078eb746f3a31c6 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/72a25f251703445f9078eb746f3a31c6 2024-11-07T09:38:22,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/72a25f251703445f9078eb746f3a31c6, entries=200, sequenceid=343, filesize=14.4 K 2024-11-07T09:38:22,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/322749f809d04d14a4dfd5f9b534de35 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/322749f809d04d14a4dfd5f9b534de35 2024-11-07T09:38:22,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/322749f809d04d14a4dfd5f9b534de35, entries=150, sequenceid=343, filesize=12.0 K 2024-11-07T09:38:22,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/7fe48d89569d409388c6fb766cb3ca6b as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/7fe48d89569d409388c6fb766cb3ca6b 2024-11-07T09:38:22,304 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/7fe48d89569d409388c6fb766cb3ca6b, entries=150, sequenceid=343, filesize=12.0 K 2024-11-07T09:38:22,305 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 57c7d380eebbfff219ec5446401d9b7d in 863ms, sequenceid=343, compaction requested=false 2024-11-07T09:38:22,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 57c7d380eebbfff219ec5446401d9b7d: 2024-11-07T09:38:22,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-07T09:38:22,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:22,417 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing 57c7d380eebbfff219ec5446401d9b7d 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-07T09:38:22,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/17f87c2145d94ff0bf2b8ba2ec5cd932 is 50, key is test_row_0/A:col10/1730972301464/Put/seqid=0 2024-11-07T09:38:22,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-07T09:38:22,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742364_1540 (size=12301) 2024-11-07T09:38:22,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 57c7d380eebbfff219ec5446401d9b7d 2024-11-07T09:38:22,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. as already flushing 2024-11-07T09:38:22,443 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T09:38:22,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:22,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972362504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:22,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:22,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:22,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:22,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:22,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972362508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:22,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:22,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:22,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:22,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:22,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38374 deadline: 1730972362510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:22,522 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:22,522 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:22,522 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:22,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:22,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972362623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:22,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:22,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:22,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:22,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:22,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972362623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:22,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972362623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:22,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:22,636 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:22,636 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:22,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:22,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:22,636 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:22,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:22,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:22,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38374 deadline: 1730972362633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:22,642 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:22,642 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:22,642 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:22,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:22,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38332 deadline: 1730972362653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:22,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:22,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:22,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:22,827 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/17f87c2145d94ff0bf2b8ba2ec5cd932 2024-11-07T09:38:22,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/64ee716ba01b45a7bf048c785cf1b159 is 50, key is test_row_0/B:col10/1730972301464/Put/seqid=0 2024-11-07T09:38:22,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742365_1541 (size=12301) 2024-11-07T09:38:22,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:22,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972362843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:22,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:22,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972362844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:22,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:22,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:22,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:22,854 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:22,854 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:22,854 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:22,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:22,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38374 deadline: 1730972362853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:22,863 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:22,863 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:22,863 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:23,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:23,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972363163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:23,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:23,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:23,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972363163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:23,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:23,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:23,176 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:23,176 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:23,176 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:23,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:23,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38374 deadline: 1730972363172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:23,182 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:23,182 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:23,183 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:23,242 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/64ee716ba01b45a7bf048c785cf1b159 2024-11-07T09:38:23,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/bd3b72ab8b0b48f4aeebc15f51dc986c is 50, key is test_row_0/C:col10/1730972301464/Put/seqid=0 2024-11-07T09:38:23,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742366_1542 (size=12301) 2024-11-07T09:38:23,321 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/bd3b72ab8b0b48f4aeebc15f51dc986c 2024-11-07T09:38:23,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/17f87c2145d94ff0bf2b8ba2ec5cd932 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/17f87c2145d94ff0bf2b8ba2ec5cd932 2024-11-07T09:38:23,329 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/17f87c2145d94ff0bf2b8ba2ec5cd932, entries=150, sequenceid=356, filesize=12.0 K 2024-11-07T09:38:23,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/64ee716ba01b45a7bf048c785cf1b159 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/64ee716ba01b45a7bf048c785cf1b159 2024-11-07T09:38:23,334 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/64ee716ba01b45a7bf048c785cf1b159, entries=150, sequenceid=356, filesize=12.0 K 2024-11-07T09:38:23,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/bd3b72ab8b0b48f4aeebc15f51dc986c as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/bd3b72ab8b0b48f4aeebc15f51dc986c 2024-11-07T09:38:23,339 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/bd3b72ab8b0b48f4aeebc15f51dc986c, entries=150, sequenceid=356, filesize=12.0 K 2024-11-07T09:38:23,340 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 57c7d380eebbfff219ec5446401d9b7d in 923ms, sequenceid=356, compaction requested=true 2024-11-07T09:38:23,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for 57c7d380eebbfff219ec5446401d9b7d: 2024-11-07T09:38:23,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:23,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-07T09:38:23,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-07T09:38:23,344 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-07T09:38:23,345 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0660 sec 2024-11-07T09:38:23,346 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 3.0710 sec 2024-11-07T09:38:23,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 57c7d380eebbfff219ec5446401d9b7d 2024-11-07T09:38:23,704 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:38:23,704 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 57c7d380eebbfff219ec5446401d9b7d 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-07T09:38:23,751 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/00480ff1477843999b722631396d2474 is 50, key is test_row_0/A:col10/1730972303698/Put/seqid=0 2024-11-07T09:38:23,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:23,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742367_1543 (size=12301) 2024-11-07T09:38:23,793 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/00480ff1477843999b722631396d2474 2024-11-07T09:38:23,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972363746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:23,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:23,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:23,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:23,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:23,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38374 deadline: 1730972363748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:23,794 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:23,794 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:23,794 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:23,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:23,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972363753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:23,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:23,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:23,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:23,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:23,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972363757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:23,814 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:23,814 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:23,814 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:23,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:23,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38332 deadline: 1730972363770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:23,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:23,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:23,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:23,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/a430823ef64844bda87ad27de54bc9fc is 50, key is test_row_0/B:col10/1730972303698/Put/seqid=0 2024-11-07T09:38:23,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742368_1544 (size=12301) 2024-11-07T09:38:23,900 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/a430823ef64844bda87ad27de54bc9fc 2024-11-07T09:38:23,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:23,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972363909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:23,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:23,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:23,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:23,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972363909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:23,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:23,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:23,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:23,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:23,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38374 deadline: 1730972363909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:23,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:23,930 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:23,930 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:23,930 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:23,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:23,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972363925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:23,937 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:23,937 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:23,937 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:23,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:23,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38332 deadline: 1730972363925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:23,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:23,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:23,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:23,941 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/43013afbb2bb43899079bbf02f0a9922 is 50, key is test_row_0/C:col10/1730972303698/Put/seqid=0 2024-11-07T09:38:23,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742369_1545 (size=12301) 2024-11-07T09:38:23,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/43013afbb2bb43899079bbf02f0a9922 2024-11-07T09:38:23,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/00480ff1477843999b722631396d2474 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/00480ff1477843999b722631396d2474 2024-11-07T09:38:24,018 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/00480ff1477843999b722631396d2474, entries=150, sequenceid=384, filesize=12.0 K 2024-11-07T09:38:24,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/a430823ef64844bda87ad27de54bc9fc as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/a430823ef64844bda87ad27de54bc9fc 2024-11-07T09:38:24,023 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/a430823ef64844bda87ad27de54bc9fc, entries=150, sequenceid=384, filesize=12.0 K 2024-11-07T09:38:24,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/43013afbb2bb43899079bbf02f0a9922 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/43013afbb2bb43899079bbf02f0a9922 2024-11-07T09:38:24,027 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/43013afbb2bb43899079bbf02f0a9922, entries=150, sequenceid=384, filesize=12.0 K 2024-11-07T09:38:24,028 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 57c7d380eebbfff219ec5446401d9b7d in 324ms, sequenceid=384, compaction requested=true 2024-11-07T09:38:24,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 57c7d380eebbfff219ec5446401d9b7d: 2024-11-07T09:38:24,028 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T09:38:24,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c7d380eebbfff219ec5446401d9b7d:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T09:38:24,030 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52428 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T09:38:24,030 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 57c7d380eebbfff219ec5446401d9b7d/A is initiating minor compaction (all files) 2024-11-07T09:38:24,030 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57c7d380eebbfff219ec5446401d9b7d/A in TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:24,030 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/acc61ee4c222475bae3be822f39ae7b1, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/72a25f251703445f9078eb746f3a31c6, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/17f87c2145d94ff0bf2b8ba2ec5cd932, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/00480ff1477843999b722631396d2474] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp, totalSize=51.2 K 2024-11-07T09:38:24,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:24,030 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting acc61ee4c222475bae3be822f39ae7b1, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1730972300186 2024-11-07T09:38:24,030 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T09:38:24,031 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72a25f251703445f9078eb746f3a31c6, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1730972300235 2024-11-07T09:38:24,031 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17f87c2145d94ff0bf2b8ba2ec5cd932, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1730972301454 2024-11-07T09:38:24,032 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T09:38:24,032 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): 57c7d380eebbfff219ec5446401d9b7d/B is initiating minor compaction (all files) 2024-11-07T09:38:24,032 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57c7d380eebbfff219ec5446401d9b7d/B in TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:24,032 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/0b3b50f45f2b442abba24afc59a90e4d, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/322749f809d04d14a4dfd5f9b534de35, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/64ee716ba01b45a7bf048c785cf1b159, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/a430823ef64844bda87ad27de54bc9fc] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp, totalSize=48.8 K 2024-11-07T09:38:24,032 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00480ff1477843999b722631396d2474, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1730972302507 2024-11-07T09:38:24,033 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b3b50f45f2b442abba24afc59a90e4d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1730972300186 2024-11-07T09:38:24,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c7d380eebbfff219ec5446401d9b7d:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T09:38:24,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:24,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57c7d380eebbfff219ec5446401d9b7d:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:38:24,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:38:24,036 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 322749f809d04d14a4dfd5f9b534de35, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1730972300235 2024-11-07T09:38:24,037 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 64ee716ba01b45a7bf048c785cf1b159, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1730972301454 2024-11-07T09:38:24,038 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting a430823ef64844bda87ad27de54bc9fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1730972302507 2024-11-07T09:38:24,062 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57c7d380eebbfff219ec5446401d9b7d#A#compaction#473 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 65.38 MB/second 2024-11-07T09:38:24,063 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/442374563cc3483c815b280d8a6fe142 is 50, key is test_row_0/A:col10/1730972303698/Put/seqid=0 2024-11-07T09:38:24,074 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57c7d380eebbfff219ec5446401d9b7d#B#compaction#474 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 65.38 MB/second 2024-11-07T09:38:24,075 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/063c261408a8441cbf9b2827c16898c1 is 50, key is test_row_0/B:col10/1730972303698/Put/seqid=0 2024-11-07T09:38:24,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742370_1546 (size=13221) 2024-11-07T09:38:24,166 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:38:24,166 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 57c7d380eebbfff219ec5446401d9b7d 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T09:38:24,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 57c7d380eebbfff219ec5446401d9b7d 2024-11-07T09:38:24,175 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/442374563cc3483c815b280d8a6fe142 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/442374563cc3483c815b280d8a6fe142 2024-11-07T09:38:24,181 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 57c7d380eebbfff219ec5446401d9b7d/A of 57c7d380eebbfff219ec5446401d9b7d into 442374563cc3483c815b280d8a6fe142(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:38:24,181 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57c7d380eebbfff219ec5446401d9b7d: 2024-11-07T09:38:24,181 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., storeName=57c7d380eebbfff219ec5446401d9b7d/A, priority=12, startTime=1730972304028; duration=0sec 2024-11-07T09:38:24,181 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:38:24,181 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c7d380eebbfff219ec5446401d9b7d:A 2024-11-07T09:38:24,181 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T09:38:24,182 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T09:38:24,182 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): 57c7d380eebbfff219ec5446401d9b7d/C is initiating minor compaction (all files) 2024-11-07T09:38:24,182 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 57c7d380eebbfff219ec5446401d9b7d/C in TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:24,183 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/77ae518b884f48758e72dcf042fdc58e, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/7fe48d89569d409388c6fb766cb3ca6b, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/bd3b72ab8b0b48f4aeebc15f51dc986c, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/43013afbb2bb43899079bbf02f0a9922] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp, totalSize=48.8 K 2024-11-07T09:38:24,183 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77ae518b884f48758e72dcf042fdc58e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1730972300186 2024-11-07T09:38:24,183 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fe48d89569d409388c6fb766cb3ca6b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1730972300235 2024-11-07T09:38:24,184 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd3b72ab8b0b48f4aeebc15f51dc986c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1730972301454 2024-11-07T09:38:24,184 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43013afbb2bb43899079bbf02f0a9922, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1730972302507 2024-11-07T09:38:24,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742371_1547 (size=13221) 2024-11-07T09:38:24,197 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/063c261408a8441cbf9b2827c16898c1 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/063c261408a8441cbf9b2827c16898c1 2024-11-07T09:38:24,202 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 57c7d380eebbfff219ec5446401d9b7d/B of 57c7d380eebbfff219ec5446401d9b7d into 063c261408a8441cbf9b2827c16898c1(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:38:24,202 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57c7d380eebbfff219ec5446401d9b7d: 2024-11-07T09:38:24,202 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., storeName=57c7d380eebbfff219ec5446401d9b7d/B, priority=12, startTime=1730972304030; duration=0sec 2024-11-07T09:38:24,202 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:24,202 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c7d380eebbfff219ec5446401d9b7d:B 2024-11-07T09:38:24,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/3641812a6433438dba5f0e537588aee2 is 50, key is test_row_0/A:col10/1730972304164/Put/seqid=0 2024-11-07T09:38:24,239 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57c7d380eebbfff219ec5446401d9b7d#C#compaction#476 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 65.38 MB/second 2024-11-07T09:38:24,239 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/a57ef13590a14d52b0e959579a57f523 is 50, key is test_row_0/C:col10/1730972303698/Put/seqid=0 2024-11-07T09:38:24,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742373_1549 (size=13221) 2024-11-07T09:38:24,323 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/a57ef13590a14d52b0e959579a57f523 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/a57ef13590a14d52b0e959579a57f523 2024-11-07T09:38:24,339 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 57c7d380eebbfff219ec5446401d9b7d/C of 57c7d380eebbfff219ec5446401d9b7d into a57ef13590a14d52b0e959579a57f523(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:38:24,339 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 57c7d380eebbfff219ec5446401d9b7d: 2024-11-07T09:38:24,339 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., storeName=57c7d380eebbfff219ec5446401d9b7d/C, priority=12, startTime=1730972304033; duration=0sec 2024-11-07T09:38:24,339 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:24,339 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57c7d380eebbfff219ec5446401d9b7d:C 2024-11-07T09:38:24,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742372_1548 (size=19621) 2024-11-07T09:38:24,345 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/3641812a6433438dba5f0e537588aee2 2024-11-07T09:38:24,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/d4329bff61a54ce393b9ff8b84034f6b is 50, key is test_row_0/B:col10/1730972304164/Put/seqid=0 2024-11-07T09:38:24,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-07T09:38:24,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:24,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972364393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:24,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:24,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:24,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:24,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972364394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:24,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:24,461 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:24,461 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:24,461 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:24,462 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:38:24,468 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T09:38:24,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees 2024-11-07T09:38:24,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:24,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972364453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:24,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:24,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:24,471 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T09:38:24,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:24,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:24,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-07T09:38:24,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38332 deadline: 1730972364460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:24,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:24,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:24,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:24,472 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T09:38:24,472 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T09:38:24,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742374_1550 (size=12301) 2024-11-07T09:38:24,508 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/d4329bff61a54ce393b9ff8b84034f6b 2024-11-07T09:38:24,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:24,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38374 deadline: 1730972364471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:24,538 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:24,539 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:24,539 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:24,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/0cc9d8c30b40495482eb48438270e1ca is 50, key is test_row_0/C:col10/1730972304164/Put/seqid=0 2024-11-07T09:38:24,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-07T09:38:24,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:24,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38288 deadline: 1730972364563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:24,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:24,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:24,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:24,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38368 deadline: 1730972364563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:24,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:24,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:24,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38318 deadline: 1730972364573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:24,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:24,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:24,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:24,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:24,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38332 deadline: 1730972364583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:24,622 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:24,622 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:24,622 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:24,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 , the old value is region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:24,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:24,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., hostname=9f1eb259a29b,42895,1730972153970, seqNum=2 because the exception is null or not the one we care about 2024-11-07T09:38:24,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-07T09:38:24,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:24,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. as already flushing 2024-11-07T09:38:24,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:24,644 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:24,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:24,647 INFO [Thread-2094 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2094. 2024-11-07T09:38:24,647 DEBUG [Thread-2094 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:38:24,647 DEBUG [Thread-2094 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:24,647 DEBUG [Thread-2094 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:24,647 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:38:24,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:24,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742375_1551 (size=12301) 2024-11-07T09:38:24,680 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/0cc9d8c30b40495482eb48438270e1ca 2024-11-07T09:38:24,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/3641812a6433438dba5f0e537588aee2 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/3641812a6433438dba5f0e537588aee2 2024-11-07T09:38:24,692 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/3641812a6433438dba5f0e537588aee2, entries=300, sequenceid=395, filesize=19.2 K 2024-11-07T09:38:24,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/d4329bff61a54ce393b9ff8b84034f6b as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/d4329bff61a54ce393b9ff8b84034f6b 2024-11-07T09:38:24,694 INFO [Thread-2096 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2096. 2024-11-07T09:38:24,694 DEBUG [Thread-2096 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:38:24,694 DEBUG [Thread-2096 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:24,694 DEBUG [Thread-2096 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:24,695 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:38:24,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/d4329bff61a54ce393b9ff8b84034f6b, entries=150, sequenceid=395, filesize=12.0 K 2024-11-07T09:38:24,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/0cc9d8c30b40495482eb48438270e1ca as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/0cc9d8c30b40495482eb48438270e1ca 2024-11-07T09:38:24,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/0cc9d8c30b40495482eb48438270e1ca, entries=150, sequenceid=395, filesize=12.0 K 2024-11-07T09:38:24,788 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 57c7d380eebbfff219ec5446401d9b7d in 621ms, sequenceid=395, compaction requested=false 2024-11-07T09:38:24,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 57c7d380eebbfff219ec5446401d9b7d: 2024-11-07T09:38:24,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-07T09:38:24,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-07T09:38:24,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:24,812 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2902): Flushing 57c7d380eebbfff219ec5446401d9b7d 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-07T09:38:24,840 INFO [Thread-2090 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2090. 2024-11-07T09:38:24,840 DEBUG [Thread-2090 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:38:24,840 DEBUG [Thread-2090 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:24,840 DEBUG [Thread-2090 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:24,840 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:38:24,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/23e03f8b92d6477097182033f9881c23 is 50, key is test_row_0/A:col10/1730972304336/Put/seqid=0 2024-11-07T09:38:24,877 INFO [Thread-2098 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2098. 2024-11-07T09:38:24,877 DEBUG [Thread-2098 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:38:24,877 DEBUG [Thread-2098 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:24,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on 57c7d380eebbfff219ec5446401d9b7d 2024-11-07T09:38:24,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. as already flushing 2024-11-07T09:38:24,877 DEBUG [Thread-2098 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:24,877 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:38:24,878 INFO [Thread-2087 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2087. 2024-11-07T09:38:24,878 DEBUG [Thread-2087 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:38:24,878 DEBUG [Thread-2087 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:24,878 DEBUG [Thread-2087 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:24,878 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:38:24,880 INFO [Thread-2092 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2092. 2024-11-07T09:38:24,880 DEBUG [Thread-2092 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:38:24,880 DEBUG [Thread-2092 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:24,880 DEBUG [Thread-2092 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:24,880 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:38:24,881 INFO [Thread-2083 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2083. 2024-11-07T09:38:24,881 DEBUG [Thread-2083 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:38:24,881 DEBUG [Thread-2083 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:24,881 DEBUG [Thread-2083 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:24,881 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:38:24,893 INFO [Thread-2079 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2079. 2024-11-07T09:38:24,893 DEBUG [Thread-2079 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:38:24,893 DEBUG [Thread-2079 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:24,893 DEBUG [Thread-2079 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:24,893 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:38:24,897 INFO [Thread-2085 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2085. 2024-11-07T09:38:24,897 DEBUG [Thread-2085 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:38:24,897 DEBUG [Thread-2085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:24,897 DEBUG [Thread-2085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:24,897 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:38:24,897 INFO [Thread-2081 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2081. 2024-11-07T09:38:24,897 DEBUG [Thread-2081 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:38:24,897 DEBUG [Thread-2081 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:24,897 DEBUG [Thread-2081 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:24,897 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:38:24,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742376_1552 (size=12301) 2024-11-07T09:38:25,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-07T09:38:25,324 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/23e03f8b92d6477097182033f9881c23 2024-11-07T09:38:25,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/d8321eed4dfc4974851c44b8ef3c1d6c is 50, key is test_row_0/B:col10/1730972304336/Put/seqid=0 2024-11-07T09:38:25,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742377_1553 (size=12301) 2024-11-07T09:38:25,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-07T09:38:25,826 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/d8321eed4dfc4974851c44b8ef3c1d6c 2024-11-07T09:38:25,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/0220d7e3bb8545b19ef2c11ec7e28232 is 50, key is test_row_0/C:col10/1730972304336/Put/seqid=0 2024-11-07T09:38:25,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742378_1554 (size=12301) 2024-11-07T09:38:26,310 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/0220d7e3bb8545b19ef2c11ec7e28232 2024-11-07T09:38:26,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/23e03f8b92d6477097182033f9881c23 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/23e03f8b92d6477097182033f9881c23 2024-11-07T09:38:26,319 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/23e03f8b92d6477097182033f9881c23, entries=150, sequenceid=424, filesize=12.0 K 2024-11-07T09:38:26,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/d8321eed4dfc4974851c44b8ef3c1d6c as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/d8321eed4dfc4974851c44b8ef3c1d6c 2024-11-07T09:38:26,323 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/d8321eed4dfc4974851c44b8ef3c1d6c, entries=150, sequenceid=424, filesize=12.0 K 2024-11-07T09:38:26,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/0220d7e3bb8545b19ef2c11ec7e28232 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/0220d7e3bb8545b19ef2c11ec7e28232 2024-11-07T09:38:26,328 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/0220d7e3bb8545b19ef2c11ec7e28232, entries=150, sequenceid=424, filesize=12.0 K 2024-11-07T09:38:26,329 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=33.54 KB/34350 for 57c7d380eebbfff219ec5446401d9b7d in 1517ms, sequenceid=424, compaction requested=true 2024-11-07T09:38:26,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2603): Flush status journal for 57c7d380eebbfff219ec5446401d9b7d: 2024-11-07T09:38:26,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:26,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=142 2024-11-07T09:38:26,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster(4169): Remote procedure done, pid=142 2024-11-07T09:38:26,333 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-07T09:38:26,333 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8590 sec 2024-11-07T09:38:26,335 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees in 1.8650 sec 2024-11-07T09:38:26,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-07T09:38:26,623 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:38:26,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-07T09:38:26,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-07T09:38:26,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-11-07T09:38:26,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-07T09:38:26,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 96 2024-11-07T09:38:26,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-07T09:38:26,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-07T09:38:26,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-07T09:38:26,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1108 2024-11-07T09:38:26,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3319 rows 2024-11-07T09:38:26,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1109 2024-11-07T09:38:26,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3322 rows 2024-11-07T09:38:26,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1110 2024-11-07T09:38:26,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3325 rows 2024-11-07T09:38:26,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1115 2024-11-07T09:38:26,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3341 rows 2024-11-07T09:38:26,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1102 2024-11-07T09:38:26,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3303 rows 2024-11-07T09:38:26,624 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T09:38:26,624 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T09:38:26,624 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:26,624 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:26,624 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:38:26,625 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-07T09:38:26,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-07T09:38:26,629 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730972306628"}]},"ts":"1730972306628"} 2024-11-07T09:38:26,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-07T09:38:26,630 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-07T09:38:26,630 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-07T09:38:26,631 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T09:38:26,633 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=57c7d380eebbfff219ec5446401d9b7d, UNASSIGN}] 2024-11-07T09:38:26,634 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=57c7d380eebbfff219ec5446401d9b7d, UNASSIGN 2024-11-07T09:38:26,635 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=57c7d380eebbfff219ec5446401d9b7d, regionState=CLOSING, regionLocation=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:26,637 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=57c7d380eebbfff219ec5446401d9b7d, UNASSIGN because future has completed 2024-11-07T09:38:26,637 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T09:38:26,637 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure 57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970}] 2024-11-07T09:38:26,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-07T09:38:26,789 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(122): Close 57c7d380eebbfff219ec5446401d9b7d 2024-11-07T09:38:26,789 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T09:38:26,789 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1722): Closing 57c7d380eebbfff219ec5446401d9b7d, disabling compactions & flushes 2024-11-07T09:38:26,789 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:26,789 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:26,789 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. after waiting 0 ms 2024-11-07T09:38:26,789 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:26,789 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(2902): Flushing 57c7d380eebbfff219ec5446401d9b7d 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-07T09:38:26,829 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/7bfdb6b62ddf47c8a7d9d6ef851a6341 is 50, key is test_row_0/A:col10/1730972304888/Put/seqid=0 2024-11-07T09:38:26,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742379_1555 (size=9857) 2024-11-07T09:38:26,882 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/7bfdb6b62ddf47c8a7d9d6ef851a6341 2024-11-07T09:38:26,888 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/d6934cb70ee443b48016e2a4cf15a188 is 50, key is test_row_0/B:col10/1730972304888/Put/seqid=0 2024-11-07T09:38:26,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742380_1556 (size=9857) 2024-11-07T09:38:26,893 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/d6934cb70ee443b48016e2a4cf15a188 2024-11-07T09:38:26,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-07T09:38:26,980 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/71609a3318354d95b9d421927dc44ba2 is 50, key is test_row_0/C:col10/1730972304888/Put/seqid=0 2024-11-07T09:38:27,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742381_1557 (size=9857) 2024-11-07T09:38:27,163 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/71609a3318354d95b9d421927dc44ba2 2024-11-07T09:38:27,167 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/A/7bfdb6b62ddf47c8a7d9d6ef851a6341 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/7bfdb6b62ddf47c8a7d9d6ef851a6341 2024-11-07T09:38:27,171 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/7bfdb6b62ddf47c8a7d9d6ef851a6341, entries=100, sequenceid=432, filesize=9.6 K 2024-11-07T09:38:27,172 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/B/d6934cb70ee443b48016e2a4cf15a188 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/d6934cb70ee443b48016e2a4cf15a188 2024-11-07T09:38:27,185 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/d6934cb70ee443b48016e2a4cf15a188, entries=100, sequenceid=432, filesize=9.6 K 2024-11-07T09:38:27,204 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/.tmp/C/71609a3318354d95b9d421927dc44ba2 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/71609a3318354d95b9d421927dc44ba2 2024-11-07T09:38:27,229 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/71609a3318354d95b9d421927dc44ba2, entries=100, sequenceid=432, filesize=9.6 K 2024-11-07T09:38:27,230 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 57c7d380eebbfff219ec5446401d9b7d in 441ms, sequenceid=432, compaction requested=true 2024-11-07T09:38:27,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-07T09:38:27,278 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/c9071ab0d6f44eb0a9904da1a3d163c4, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/1d23c12a75334a289a845eead6958492, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/36c3401cdec544c88b50cccc13cdf582, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/5f31c2f4da0c4337a82a71179a52ebb5, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/b903f54c3043437fa9be0193adeb9633, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/5c3445fd49d7439084dede48f0514d3a, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/c1d2a725fbbe4de991e24b0a940cfbe1, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/299db1b4f43d45a5a70cc5a54d08fd53, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/dc8dbbfa484246bc9c69ab3beb84e5f4, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/b04455418b3046ba90a54545f1957062, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/d04ed992507642d7a03d0ce61286c653, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/557a5b5252ae4cd2b405dd9c745f3220, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/4e555b765d9b4e8e9493db6df0227273, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/2711b4ae4b504926af5ffc4b2d80a380, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/34590c4672184e3198e8b8938a98746e, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/1ce0912af74a4ecaa3749088986cefe0, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/27f9b044fb2c4102b18b29c10eea7307, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/82dd2474e43c4d12828d111c8bff84c0, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/3281bf9b98e8493bbc72919faf3ed1c4, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/0d2fb9f52fd84848b3581cdf352bc997, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/026f839b06f54dfdb157a97853b80221, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/08949a90ae85487fb167a877e552494c, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/8ac30e0476c2438480538d5dec9a7536, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/acc61ee4c222475bae3be822f39ae7b1, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/72a25f251703445f9078eb746f3a31c6, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/17f87c2145d94ff0bf2b8ba2ec5cd932, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/00480ff1477843999b722631396d2474] to archive 2024-11-07T09:38:27,328 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T09:38:27,344 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/c9071ab0d6f44eb0a9904da1a3d163c4 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/c9071ab0d6f44eb0a9904da1a3d163c4 2024-11-07T09:38:27,356 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/1d23c12a75334a289a845eead6958492 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/1d23c12a75334a289a845eead6958492 2024-11-07T09:38:27,367 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/36c3401cdec544c88b50cccc13cdf582 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/36c3401cdec544c88b50cccc13cdf582 2024-11-07T09:38:27,371 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/5f31c2f4da0c4337a82a71179a52ebb5 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/5f31c2f4da0c4337a82a71179a52ebb5 2024-11-07T09:38:27,384 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/b903f54c3043437fa9be0193adeb9633 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/b903f54c3043437fa9be0193adeb9633 2024-11-07T09:38:27,428 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/5c3445fd49d7439084dede48f0514d3a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/5c3445fd49d7439084dede48f0514d3a 2024-11-07T09:38:27,460 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/c1d2a725fbbe4de991e24b0a940cfbe1 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/c1d2a725fbbe4de991e24b0a940cfbe1 2024-11-07T09:38:27,469 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/299db1b4f43d45a5a70cc5a54d08fd53 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/299db1b4f43d45a5a70cc5a54d08fd53 2024-11-07T09:38:27,480 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/dc8dbbfa484246bc9c69ab3beb84e5f4 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/dc8dbbfa484246bc9c69ab3beb84e5f4 2024-11-07T09:38:27,496 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/b04455418b3046ba90a54545f1957062 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/b04455418b3046ba90a54545f1957062 2024-11-07T09:38:27,500 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/d04ed992507642d7a03d0ce61286c653 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/d04ed992507642d7a03d0ce61286c653 2024-11-07T09:38:27,505 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/557a5b5252ae4cd2b405dd9c745f3220 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/557a5b5252ae4cd2b405dd9c745f3220 2024-11-07T09:38:27,508 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/4e555b765d9b4e8e9493db6df0227273 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/4e555b765d9b4e8e9493db6df0227273 2024-11-07T09:38:27,527 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/2711b4ae4b504926af5ffc4b2d80a380 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/2711b4ae4b504926af5ffc4b2d80a380 2024-11-07T09:38:27,533 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/34590c4672184e3198e8b8938a98746e to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/34590c4672184e3198e8b8938a98746e 2024-11-07T09:38:27,534 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/1ce0912af74a4ecaa3749088986cefe0 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/1ce0912af74a4ecaa3749088986cefe0 2024-11-07T09:38:27,535 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/27f9b044fb2c4102b18b29c10eea7307 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/27f9b044fb2c4102b18b29c10eea7307 2024-11-07T09:38:27,536 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/82dd2474e43c4d12828d111c8bff84c0 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/82dd2474e43c4d12828d111c8bff84c0 2024-11-07T09:38:27,558 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/3281bf9b98e8493bbc72919faf3ed1c4 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/3281bf9b98e8493bbc72919faf3ed1c4 2024-11-07T09:38:27,571 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/0d2fb9f52fd84848b3581cdf352bc997 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/0d2fb9f52fd84848b3581cdf352bc997 2024-11-07T09:38:27,584 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/026f839b06f54dfdb157a97853b80221 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/026f839b06f54dfdb157a97853b80221 2024-11-07T09:38:27,593 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/08949a90ae85487fb167a877e552494c to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/08949a90ae85487fb167a877e552494c 2024-11-07T09:38:27,594 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/8ac30e0476c2438480538d5dec9a7536 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/8ac30e0476c2438480538d5dec9a7536 2024-11-07T09:38:27,595 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/acc61ee4c222475bae3be822f39ae7b1 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/acc61ee4c222475bae3be822f39ae7b1 2024-11-07T09:38:27,597 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/72a25f251703445f9078eb746f3a31c6 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/72a25f251703445f9078eb746f3a31c6 2024-11-07T09:38:27,598 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/17f87c2145d94ff0bf2b8ba2ec5cd932 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/17f87c2145d94ff0bf2b8ba2ec5cd932 2024-11-07T09:38:27,599 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/00480ff1477843999b722631396d2474 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/00480ff1477843999b722631396d2474 2024-11-07T09:38:27,631 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/5aa7244bae9c4cebac41dd6a47a1ac90, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/bdf5d4e74f5f4b54afd978ad62194530, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/37529645f7f04844a918c3794e5951fe, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/c15b08bd21874ffd9048738c9782e66c, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/efacfcd7e7604169b927e3980b3fc2d3, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/81ac7ef7c02949c1bb9892ef48979f5f, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/0a3609c4b4204a3cac30dcce2f63b859, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/a78b7cac61dd4280bd24419246f5c7d6, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/532a56968230480db6dae542994fdbfe, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/ac66b68b296b4a4fa3902a9a33401edf, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/aa9ca9750622482f8337923786a751d7, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/007ec9fdae354c9da7dea77106181ae4, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/8fc3cdf2547a4a618108cc922c649073, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/1408287bb2eb4fc8b7b59cf835916f22, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/836ba69fe46149429e5b0552a696b8b6, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/bd8d07ec0bfe46daa642847d5bfd468a, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/6f573031dc2c4e08a3ca4c499dd076a0, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/edf2de82d12f4a87af6725193630baa8, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/f788b378995249e68b70889f0e9fad59, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/6e78229fb200475694b74ddf6bddb954, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/e249c89857fe4158899d8368be79153c, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/da1851007fe047898657afd2b6a2df04, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/0b3b50f45f2b442abba24afc59a90e4d, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/c3756d6adea94a7fbecb14da86a08e3c, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/322749f809d04d14a4dfd5f9b534de35, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/64ee716ba01b45a7bf048c785cf1b159, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/a430823ef64844bda87ad27de54bc9fc] to archive 2024-11-07T09:38:27,633 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T09:38:27,638 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/5aa7244bae9c4cebac41dd6a47a1ac90 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/5aa7244bae9c4cebac41dd6a47a1ac90 2024-11-07T09:38:27,651 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/bdf5d4e74f5f4b54afd978ad62194530 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/bdf5d4e74f5f4b54afd978ad62194530 2024-11-07T09:38:27,652 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/37529645f7f04844a918c3794e5951fe to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/37529645f7f04844a918c3794e5951fe 2024-11-07T09:38:27,663 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/c15b08bd21874ffd9048738c9782e66c to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/c15b08bd21874ffd9048738c9782e66c 2024-11-07T09:38:27,665 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/efacfcd7e7604169b927e3980b3fc2d3 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/efacfcd7e7604169b927e3980b3fc2d3 2024-11-07T09:38:27,666 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/81ac7ef7c02949c1bb9892ef48979f5f to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/81ac7ef7c02949c1bb9892ef48979f5f 2024-11-07T09:38:27,667 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/0a3609c4b4204a3cac30dcce2f63b859 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/0a3609c4b4204a3cac30dcce2f63b859 2024-11-07T09:38:27,672 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/a78b7cac61dd4280bd24419246f5c7d6 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/a78b7cac61dd4280bd24419246f5c7d6 2024-11-07T09:38:27,674 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/532a56968230480db6dae542994fdbfe to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/532a56968230480db6dae542994fdbfe 2024-11-07T09:38:27,678 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/ac66b68b296b4a4fa3902a9a33401edf to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/ac66b68b296b4a4fa3902a9a33401edf 2024-11-07T09:38:27,684 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/aa9ca9750622482f8337923786a751d7 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/aa9ca9750622482f8337923786a751d7 2024-11-07T09:38:27,692 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/007ec9fdae354c9da7dea77106181ae4 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/007ec9fdae354c9da7dea77106181ae4 2024-11-07T09:38:27,698 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/8fc3cdf2547a4a618108cc922c649073 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/8fc3cdf2547a4a618108cc922c649073 2024-11-07T09:38:27,708 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/1408287bb2eb4fc8b7b59cf835916f22 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/1408287bb2eb4fc8b7b59cf835916f22 2024-11-07T09:38:27,714 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/836ba69fe46149429e5b0552a696b8b6 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/836ba69fe46149429e5b0552a696b8b6 2024-11-07T09:38:27,716 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/bd8d07ec0bfe46daa642847d5bfd468a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/bd8d07ec0bfe46daa642847d5bfd468a 2024-11-07T09:38:27,718 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/6f573031dc2c4e08a3ca4c499dd076a0 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/6f573031dc2c4e08a3ca4c499dd076a0 2024-11-07T09:38:27,722 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/edf2de82d12f4a87af6725193630baa8 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/edf2de82d12f4a87af6725193630baa8 2024-11-07T09:38:27,732 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/f788b378995249e68b70889f0e9fad59 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/f788b378995249e68b70889f0e9fad59 2024-11-07T09:38:27,740 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/6e78229fb200475694b74ddf6bddb954 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/6e78229fb200475694b74ddf6bddb954 2024-11-07T09:38:27,742 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/e249c89857fe4158899d8368be79153c to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/e249c89857fe4158899d8368be79153c 2024-11-07T09:38:27,752 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/da1851007fe047898657afd2b6a2df04 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/da1851007fe047898657afd2b6a2df04 2024-11-07T09:38:27,756 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/0b3b50f45f2b442abba24afc59a90e4d to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/0b3b50f45f2b442abba24afc59a90e4d 2024-11-07T09:38:27,761 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/c3756d6adea94a7fbecb14da86a08e3c to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/c3756d6adea94a7fbecb14da86a08e3c 2024-11-07T09:38:27,762 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/322749f809d04d14a4dfd5f9b534de35 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/322749f809d04d14a4dfd5f9b534de35 2024-11-07T09:38:27,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-07T09:38:27,763 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/64ee716ba01b45a7bf048c785cf1b159 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/64ee716ba01b45a7bf048c785cf1b159 2024-11-07T09:38:27,764 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/a430823ef64844bda87ad27de54bc9fc to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/a430823ef64844bda87ad27de54bc9fc 2024-11-07T09:38:27,803 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/76b11018c64e4d41a6fbe8c677fb9b56, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/94a651b42b36474bb83956956acb1cfd, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/3465c6bc8bae407ab85d499a5480b2b6, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/67f23380cb5443e6ba28ebacb82f3865, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/7edbf7b0310f4432a0f618509df25c17, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/b3a9e8bd52f146039305d7fbe2576dd2, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/1e7c5b716d26422e9d63af09ed84b7d0, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/3c5474de56634d2eaef2b4a63f9f7dd0, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/a81ec63a2fad45518a89cd6241217ba1, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/b7d2d5ed9c634995b0b9734abad22bbb, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/a9955098ac29427fa2a1917d7529fa0a, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/58fc255aa42a45f1beaef33cf7e37161, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/3aafd6af766f4879875a5cba8a64f224, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/e863f47d2fe6449cba3f48a66ecb4524, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/854b9b595c0c417fbe03f4ee42bec207, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/3d0220c4d9e14c9c97e01b683b7af954, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/4907f3e5fd7d4921bccfdc2cf3e1a78b, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/8fbb5bddeaa543dd9f68f9c51ed16c4b, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/598b178fdb5a42b188d1c9737bca1d44, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/eb0fa217ebb248dd8fe1d0b42dc570c7, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/bdb9358f627944929075729b7b07f865, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/4288bf1548444502bdbf819c863dbeb4, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/77ae518b884f48758e72dcf042fdc58e, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/e5cc5bc122cf4b61af1ddb5ab0f90935, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/7fe48d89569d409388c6fb766cb3ca6b, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/bd3b72ab8b0b48f4aeebc15f51dc986c, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/43013afbb2bb43899079bbf02f0a9922] to archive 2024-11-07T09:38:27,804 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T09:38:27,826 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/76b11018c64e4d41a6fbe8c677fb9b56 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/76b11018c64e4d41a6fbe8c677fb9b56 2024-11-07T09:38:27,837 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/94a651b42b36474bb83956956acb1cfd to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/94a651b42b36474bb83956956acb1cfd 2024-11-07T09:38:27,856 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/3465c6bc8bae407ab85d499a5480b2b6 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/3465c6bc8bae407ab85d499a5480b2b6 2024-11-07T09:38:27,868 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/67f23380cb5443e6ba28ebacb82f3865 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/67f23380cb5443e6ba28ebacb82f3865 2024-11-07T09:38:27,869 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/7edbf7b0310f4432a0f618509df25c17 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/7edbf7b0310f4432a0f618509df25c17 2024-11-07T09:38:27,871 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/b3a9e8bd52f146039305d7fbe2576dd2 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/b3a9e8bd52f146039305d7fbe2576dd2 2024-11-07T09:38:27,872 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/1e7c5b716d26422e9d63af09ed84b7d0 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/1e7c5b716d26422e9d63af09ed84b7d0 2024-11-07T09:38:27,876 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/3c5474de56634d2eaef2b4a63f9f7dd0 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/3c5474de56634d2eaef2b4a63f9f7dd0 2024-11-07T09:38:27,879 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/a81ec63a2fad45518a89cd6241217ba1 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/a81ec63a2fad45518a89cd6241217ba1 2024-11-07T09:38:27,886 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/b7d2d5ed9c634995b0b9734abad22bbb to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/b7d2d5ed9c634995b0b9734abad22bbb 2024-11-07T09:38:27,902 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/a9955098ac29427fa2a1917d7529fa0a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/a9955098ac29427fa2a1917d7529fa0a 2024-11-07T09:38:27,913 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/58fc255aa42a45f1beaef33cf7e37161 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/58fc255aa42a45f1beaef33cf7e37161 2024-11-07T09:38:27,944 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/3aafd6af766f4879875a5cba8a64f224 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/3aafd6af766f4879875a5cba8a64f224 2024-11-07T09:38:27,958 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/e863f47d2fe6449cba3f48a66ecb4524 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/e863f47d2fe6449cba3f48a66ecb4524 2024-11-07T09:38:27,966 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/854b9b595c0c417fbe03f4ee42bec207 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/854b9b595c0c417fbe03f4ee42bec207 2024-11-07T09:38:27,988 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/3d0220c4d9e14c9c97e01b683b7af954 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/3d0220c4d9e14c9c97e01b683b7af954 2024-11-07T09:38:28,000 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/4907f3e5fd7d4921bccfdc2cf3e1a78b to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/4907f3e5fd7d4921bccfdc2cf3e1a78b 2024-11-07T09:38:28,009 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/8fbb5bddeaa543dd9f68f9c51ed16c4b to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/8fbb5bddeaa543dd9f68f9c51ed16c4b 2024-11-07T09:38:28,011 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/598b178fdb5a42b188d1c9737bca1d44 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/598b178fdb5a42b188d1c9737bca1d44 2024-11-07T09:38:28,012 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/eb0fa217ebb248dd8fe1d0b42dc570c7 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/eb0fa217ebb248dd8fe1d0b42dc570c7 2024-11-07T09:38:28,013 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/bdb9358f627944929075729b7b07f865 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/bdb9358f627944929075729b7b07f865 2024-11-07T09:38:28,014 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/4288bf1548444502bdbf819c863dbeb4 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/4288bf1548444502bdbf819c863dbeb4 2024-11-07T09:38:28,018 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/77ae518b884f48758e72dcf042fdc58e to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/77ae518b884f48758e72dcf042fdc58e 2024-11-07T09:38:28,038 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/e5cc5bc122cf4b61af1ddb5ab0f90935 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/e5cc5bc122cf4b61af1ddb5ab0f90935 2024-11-07T09:38:28,048 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/7fe48d89569d409388c6fb766cb3ca6b to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/7fe48d89569d409388c6fb766cb3ca6b 2024-11-07T09:38:28,064 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/bd3b72ab8b0b48f4aeebc15f51dc986c to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/bd3b72ab8b0b48f4aeebc15f51dc986c 2024-11-07T09:38:28,072 DEBUG [StoreCloser-TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/43013afbb2bb43899079bbf02f0a9922 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/43013afbb2bb43899079bbf02f0a9922 2024-11-07T09:38:28,168 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/recovered.edits/435.seqid, newMaxSeqId=435, maxSeqId=1 2024-11-07T09:38:28,169 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. 2024-11-07T09:38:28,169 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1676): Region close journal for 57c7d380eebbfff219ec5446401d9b7d: Waiting for close lock at 1730972306789Running coprocessor pre-close hooks at 1730972306789Disabling compacts and flushes for region at 1730972306789Disabling writes for close at 1730972306789Obtaining lock to block concurrent updates at 1730972306789Preparing flush snapshotting stores in 57c7d380eebbfff219ec5446401d9b7d at 1730972306789Finished memstore snapshotting TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1730972306790 (+1 ms)Flushing stores of TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d. at 1730972306791 (+1 ms)Flushing 57c7d380eebbfff219ec5446401d9b7d/A: creating writer at 1730972306791Flushing 57c7d380eebbfff219ec5446401d9b7d/A: appending metadata at 1730972306829 (+38 ms)Flushing 57c7d380eebbfff219ec5446401d9b7d/A: closing flushed file at 1730972306829Flushing 57c7d380eebbfff219ec5446401d9b7d/B: creating writer at 1730972306885 (+56 ms)Flushing 57c7d380eebbfff219ec5446401d9b7d/B: appending metadata at 1730972306888 (+3 ms)Flushing 57c7d380eebbfff219ec5446401d9b7d/B: closing flushed file at 1730972306888Flushing 57c7d380eebbfff219ec5446401d9b7d/C: creating writer at 1730972306896 (+8 ms)Flushing 57c7d380eebbfff219ec5446401d9b7d/C: appending metadata at 1730972306980 (+84 ms)Flushing 57c7d380eebbfff219ec5446401d9b7d/C: closing flushed file at 1730972306980Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@30e99217: reopening flushed file at 1730972307166 (+186 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@61117d33: reopening flushed file at 1730972307171 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7413e842: reopening flushed file at 1730972307185 (+14 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 57c7d380eebbfff219ec5446401d9b7d in 441ms, sequenceid=432, compaction requested=true at 1730972307231 (+46 ms)Writing region close event to WAL at 1730972308114 (+883 ms)Running coprocessor post-close hooks at 1730972308169 (+55 ms)Closed at 1730972308169 2024-11-07T09:38:28,173 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=57c7d380eebbfff219ec5446401d9b7d, regionState=CLOSED 2024-11-07T09:38:28,174 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(157): Closed 57c7d380eebbfff219ec5446401d9b7d 2024-11-07T09:38:28,174 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure 57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 because future has completed 2024-11-07T09:38:28,177 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-07T09:38:28,177 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; CloseRegionProcedure 57c7d380eebbfff219ec5446401d9b7d, server=9f1eb259a29b,42895,1730972153970 in 1.5380 sec 2024-11-07T09:38:28,179 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-07T09:38:28,179 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=57c7d380eebbfff219ec5446401d9b7d, UNASSIGN in 1.5440 sec 2024-11-07T09:38:28,182 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-07T09:38:28,182 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5490 sec 2024-11-07T09:38:28,184 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730972308184"}]},"ts":"1730972308184"} 2024-11-07T09:38:28,185 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-07T09:38:28,185 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-07T09:38:28,187 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5610 sec 2024-11-07T09:38:28,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-07T09:38:28,783 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:38:28,784 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-07T09:38:28,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T09:38:28,790 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=147, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T09:38:28,791 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=147, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T09:38:28,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-07T09:38:28,824 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d 2024-11-07T09:38:28,840 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A, FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B, FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C, FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/recovered.edits] 2024-11-07T09:38:28,844 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/23e03f8b92d6477097182033f9881c23 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/23e03f8b92d6477097182033f9881c23 2024-11-07T09:38:28,845 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/3641812a6433438dba5f0e537588aee2 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/3641812a6433438dba5f0e537588aee2 2024-11-07T09:38:28,846 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/442374563cc3483c815b280d8a6fe142 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/442374563cc3483c815b280d8a6fe142 2024-11-07T09:38:28,847 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/7bfdb6b62ddf47c8a7d9d6ef851a6341 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/A/7bfdb6b62ddf47c8a7d9d6ef851a6341 2024-11-07T09:38:28,850 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/063c261408a8441cbf9b2827c16898c1 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/063c261408a8441cbf9b2827c16898c1 2024-11-07T09:38:28,879 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/d4329bff61a54ce393b9ff8b84034f6b to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/d4329bff61a54ce393b9ff8b84034f6b 2024-11-07T09:38:28,880 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/d6934cb70ee443b48016e2a4cf15a188 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/d6934cb70ee443b48016e2a4cf15a188 2024-11-07T09:38:28,881 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/d8321eed4dfc4974851c44b8ef3c1d6c to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/B/d8321eed4dfc4974851c44b8ef3c1d6c 2024-11-07T09:38:28,882 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/0220d7e3bb8545b19ef2c11ec7e28232 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/0220d7e3bb8545b19ef2c11ec7e28232 2024-11-07T09:38:28,883 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/0cc9d8c30b40495482eb48438270e1ca to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/0cc9d8c30b40495482eb48438270e1ca 2024-11-07T09:38:28,884 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/71609a3318354d95b9d421927dc44ba2 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/71609a3318354d95b9d421927dc44ba2 2024-11-07T09:38:28,890 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/a57ef13590a14d52b0e959579a57f523 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/C/a57ef13590a14d52b0e959579a57f523 2024-11-07T09:38:28,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-07T09:38:28,947 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/recovered.edits/435.seqid to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d/recovered.edits/435.seqid 2024-11-07T09:38:28,964 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/57c7d380eebbfff219ec5446401d9b7d 2024-11-07T09:38:28,964 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-07T09:38:28,967 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=147, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T09:38:28,970 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-07T09:38:28,973 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-07T09:38:28,974 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=147, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T09:38:28,974 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-07T09:38:28,974 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1730972308974"}]},"ts":"9223372036854775807"} 2024-11-07T09:38:28,977 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-07T09:38:28,977 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 57c7d380eebbfff219ec5446401d9b7d, NAME => 'TestAcidGuarantees,,1730972283458.57c7d380eebbfff219ec5446401d9b7d.', STARTKEY => '', ENDKEY => ''}] 2024-11-07T09:38:28,977 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-07T09:38:28,977 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1730972308977"}]},"ts":"9223372036854775807"} 2024-11-07T09:38:28,979 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-07T09:38:28,980 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=147, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T09:38:28,981 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 196 msec 2024-11-07T09:38:29,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-07T09:38:29,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-07T09:38:29,134 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:38:29,214 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=240 (was 242), OpenFileDescriptor=447 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1094 (was 372) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4281 (was 4352) 2024-11-07T09:38:29,291 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=240, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=1094, ProcessCount=11, AvailableMemoryMB=4280 2024-11-07T09:38:29,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T09:38:29,293 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T09:38:29,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-07T09:38:29,295 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T09:38:29,296 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:29,298 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T09:38:29,298 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 148 2024-11-07T09:38:29,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-07T09:38:29,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742382_1558 (size=959) 2024-11-07T09:38:29,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-07T09:38:29,446 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260 2024-11-07T09:38:29,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-07T09:38:29,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742383_1559 (size=53) 2024-11-07T09:38:29,757 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T09:38:29,757 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e4cc6107007e595af5a0df7b5bc3711a, disabling compactions & flushes 2024-11-07T09:38:29,758 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:29,758 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:29,758 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. after waiting 0 ms 2024-11-07T09:38:29,758 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:29,758 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:29,758 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e4cc6107007e595af5a0df7b5bc3711a: Waiting for close lock at 1730972309757Disabling compacts and flushes for region at 1730972309757Disabling writes for close at 1730972309758 (+1 ms)Writing region close event to WAL at 1730972309758Closed at 1730972309758 2024-11-07T09:38:29,759 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T09:38:29,759 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1730972309759"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730972309759"}]},"ts":"1730972309759"} 2024-11-07T09:38:29,764 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-07T09:38:29,766 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T09:38:29,766 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730972309766"}]},"ts":"1730972309766"} 2024-11-07T09:38:29,772 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-07T09:38:29,773 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e4cc6107007e595af5a0df7b5bc3711a, ASSIGN}] 2024-11-07T09:38:29,774 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e4cc6107007e595af5a0df7b5bc3711a, ASSIGN 2024-11-07T09:38:29,775 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e4cc6107007e595af5a0df7b5bc3711a, ASSIGN; state=OFFLINE, location=9f1eb259a29b,42895,1730972153970; forceNewPlan=false, retain=false 2024-11-07T09:38:29,926 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=e4cc6107007e595af5a0df7b5bc3711a, regionState=OPENING, regionLocation=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:29,928 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e4cc6107007e595af5a0df7b5bc3711a, ASSIGN because future has completed 2024-11-07T09:38:29,929 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970}] 2024-11-07T09:38:29,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-07T09:38:30,087 INFO [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:30,088 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(7752): Opening region: {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} 2024-11-07T09:38:30,088 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:30,088 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T09:38:30,088 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(7794): checking encryption for e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:30,088 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(7797): checking classloading for e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:30,118 INFO [StoreOpener-e4cc6107007e595af5a0df7b5bc3711a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:30,129 INFO [StoreOpener-e4cc6107007e595af5a0df7b5bc3711a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e4cc6107007e595af5a0df7b5bc3711a columnFamilyName A 2024-11-07T09:38:30,129 DEBUG [StoreOpener-e4cc6107007e595af5a0df7b5bc3711a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:30,129 INFO [StoreOpener-e4cc6107007e595af5a0df7b5bc3711a-1 {}] regionserver.HStore(327): Store=e4cc6107007e595af5a0df7b5bc3711a/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T09:38:30,129 INFO [StoreOpener-e4cc6107007e595af5a0df7b5bc3711a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:30,131 INFO [StoreOpener-e4cc6107007e595af5a0df7b5bc3711a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e4cc6107007e595af5a0df7b5bc3711a columnFamilyName B 2024-11-07T09:38:30,131 DEBUG [StoreOpener-e4cc6107007e595af5a0df7b5bc3711a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:30,131 INFO [StoreOpener-e4cc6107007e595af5a0df7b5bc3711a-1 {}] regionserver.HStore(327): Store=e4cc6107007e595af5a0df7b5bc3711a/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T09:38:30,132 INFO [StoreOpener-e4cc6107007e595af5a0df7b5bc3711a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:30,133 INFO [StoreOpener-e4cc6107007e595af5a0df7b5bc3711a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e4cc6107007e595af5a0df7b5bc3711a columnFamilyName C 2024-11-07T09:38:30,133 DEBUG [StoreOpener-e4cc6107007e595af5a0df7b5bc3711a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:30,133 INFO [StoreOpener-e4cc6107007e595af5a0df7b5bc3711a-1 {}] regionserver.HStore(327): Store=e4cc6107007e595af5a0df7b5bc3711a/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T09:38:30,133 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1038): replaying wal for e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:30,134 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:30,134 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:30,137 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1048): stopping wal replay for e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:30,137 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1060): Cleaning up temporary data for e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:30,137 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T09:38:30,139 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1093): writing seq id for e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:30,263 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T09:38:30,264 INFO [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1114): Opened e4cc6107007e595af5a0df7b5bc3711a; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67714834, jitterRate=0.009029656648635864}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T09:38:30,264 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:30,265 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1006): Region open journal for e4cc6107007e595af5a0df7b5bc3711a: Running coprocessor pre-open hook at 1730972310088Writing region info on filesystem at 1730972310088Initializing all the Stores at 1730972310089 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730972310089Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730972310118 (+29 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730972310118Cleaning up temporary data from old regions at 1730972310137 (+19 ms)Running coprocessor post-open hooks at 1730972310264 (+127 ms)Region opened successfully at 1730972310265 (+1 ms) 2024-11-07T09:38:30,269 INFO [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., pid=150, masterSystemTime=1730972310081 2024-11-07T09:38:30,271 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:30,271 INFO [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:30,271 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=e4cc6107007e595af5a0df7b5bc3711a, regionState=OPEN, openSeqNum=2, regionLocation=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:30,274 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=150, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 because future has completed 2024-11-07T09:38:30,276 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-07T09:38:30,276 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; OpenRegionProcedure e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 in 345 msec 2024-11-07T09:38:30,282 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-07T09:38:30,282 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e4cc6107007e595af5a0df7b5bc3711a, ASSIGN in 504 msec 2024-11-07T09:38:30,287 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T09:38:30,288 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730972310288"}]},"ts":"1730972310288"} 2024-11-07T09:38:30,289 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-07T09:38:30,290 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T09:38:30,292 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 997 msec 2024-11-07T09:38:30,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-07T09:38:30,444 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:38:30,498 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4826bbfc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:30,498 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:38:30,499 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:38:30,501 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:38:30,501 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:38:30,502 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:38:30,502 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f5b2025, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:30,502 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:38:30,502 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:38:30,502 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:30,504 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44810, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:38:30,505 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@659843d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:30,505 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:38:30,506 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:38:30,506 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:38:30,508 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46900, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:38:30,509 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T09:38:30,510 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 9f1eb259a29b,41689,1730972153310 2024-11-07T09:38:30,510 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4d13cc38 2024-11-07T09:38:30,511 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T09:38:30,512 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44816, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T09:38:30,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T09:38:30,518 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T09:38:30,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=151, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-07T09:38:30,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742384_1560 (size=995) 2024-11-07T09:38:30,697 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-07T09:38:30,697 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-07T09:38:30,700 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T09:38:30,705 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e4cc6107007e595af5a0df7b5bc3711a, REOPEN/MOVE}] 2024-11-07T09:38:30,707 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e4cc6107007e595af5a0df7b5bc3711a, REOPEN/MOVE 2024-11-07T09:38:30,709 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=e4cc6107007e595af5a0df7b5bc3711a, regionState=CLOSING, regionLocation=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:30,710 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e4cc6107007e595af5a0df7b5bc3711a, REOPEN/MOVE because future has completed 2024-11-07T09:38:30,710 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T09:38:30,711 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE, hasLock=false; CloseRegionProcedure e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970}] 2024-11-07T09:38:30,870 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] handler.UnassignRegionHandler(122): Close e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:30,870 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T09:38:30,870 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1722): Closing e4cc6107007e595af5a0df7b5bc3711a, disabling compactions & flushes 2024-11-07T09:38:30,871 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:30,871 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:30,871 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. after waiting 0 ms 2024-11-07T09:38:30,871 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:30,885 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T09:38:30,886 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:30,886 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1676): Region close journal for e4cc6107007e595af5a0df7b5bc3711a: Waiting for close lock at 1730972310870Running coprocessor pre-close hooks at 1730972310870Disabling compacts and flushes for region at 1730972310870Disabling writes for close at 1730972310871 (+1 ms)Writing region close event to WAL at 1730972310874 (+3 ms)Running coprocessor post-close hooks at 1730972310886 (+12 ms)Closed at 1730972310886 2024-11-07T09:38:30,886 WARN [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegionServer(3299): Not adding moved region record: e4cc6107007e595af5a0df7b5bc3711a to self. 2024-11-07T09:38:30,888 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] handler.UnassignRegionHandler(157): Closed e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:30,889 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=e4cc6107007e595af5a0df7b5bc3711a, regionState=CLOSED 2024-11-07T09:38:30,891 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=154, ppid=153, state=RUNNABLE, hasLock=false; CloseRegionProcedure e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 because future has completed 2024-11-07T09:38:30,893 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=154, resume processing ppid=153 2024-11-07T09:38:30,894 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, ppid=153, state=SUCCESS, hasLock=false; CloseRegionProcedure e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 in 181 msec 2024-11-07T09:38:30,898 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e4cc6107007e595af5a0df7b5bc3711a, REOPEN/MOVE; state=CLOSED, location=9f1eb259a29b,42895,1730972153970; forceNewPlan=false, retain=true 2024-11-07T09:38:31,049 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=e4cc6107007e595af5a0df7b5bc3711a, regionState=OPENING, regionLocation=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:31,051 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e4cc6107007e595af5a0df7b5bc3711a, REOPEN/MOVE because future has completed 2024-11-07T09:38:31,053 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=153, state=RUNNABLE, hasLock=false; OpenRegionProcedure e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970}] 2024-11-07T09:38:31,213 INFO [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:31,213 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(7752): Opening region: {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} 2024-11-07T09:38:31,214 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:31,214 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T09:38:31,214 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(7794): checking encryption for e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:31,214 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(7797): checking classloading for e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:31,262 INFO [StoreOpener-e4cc6107007e595af5a0df7b5bc3711a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:31,296 INFO [StoreOpener-e4cc6107007e595af5a0df7b5bc3711a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e4cc6107007e595af5a0df7b5bc3711a columnFamilyName A 2024-11-07T09:38:31,298 DEBUG [StoreOpener-e4cc6107007e595af5a0df7b5bc3711a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:31,298 INFO [StoreOpener-e4cc6107007e595af5a0df7b5bc3711a-1 {}] regionserver.HStore(327): Store=e4cc6107007e595af5a0df7b5bc3711a/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T09:38:31,299 INFO [StoreOpener-e4cc6107007e595af5a0df7b5bc3711a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:31,299 INFO [StoreOpener-e4cc6107007e595af5a0df7b5bc3711a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e4cc6107007e595af5a0df7b5bc3711a columnFamilyName B 2024-11-07T09:38:31,299 DEBUG [StoreOpener-e4cc6107007e595af5a0df7b5bc3711a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:31,300 INFO [StoreOpener-e4cc6107007e595af5a0df7b5bc3711a-1 {}] regionserver.HStore(327): Store=e4cc6107007e595af5a0df7b5bc3711a/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T09:38:31,300 INFO [StoreOpener-e4cc6107007e595af5a0df7b5bc3711a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:31,301 INFO [StoreOpener-e4cc6107007e595af5a0df7b5bc3711a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e4cc6107007e595af5a0df7b5bc3711a columnFamilyName C 2024-11-07T09:38:31,301 DEBUG [StoreOpener-e4cc6107007e595af5a0df7b5bc3711a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:31,304 INFO [StoreOpener-e4cc6107007e595af5a0df7b5bc3711a-1 {}] regionserver.HStore(327): Store=e4cc6107007e595af5a0df7b5bc3711a/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T09:38:31,305 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1038): replaying wal for e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:31,305 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:31,306 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:31,307 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1048): stopping wal replay for e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:31,307 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1060): Cleaning up temporary data for e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:31,308 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T09:38:31,309 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1093): writing seq id for e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:31,310 INFO [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1114): Opened e4cc6107007e595af5a0df7b5bc3711a; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70593950, jitterRate=0.05193182826042175}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T09:38:31,310 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:31,311 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1006): Region open journal for e4cc6107007e595af5a0df7b5bc3711a: Running coprocessor pre-open hook at 1730972311214Writing region info on filesystem at 1730972311214Initializing all the Stores at 1730972311215 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730972311216 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730972311262 (+46 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730972311262Cleaning up temporary data from old regions at 1730972311307 (+45 ms)Running coprocessor post-open hooks at 1730972311310 (+3 ms)Region opened successfully at 1730972311311 (+1 ms) 2024-11-07T09:38:31,312 INFO [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., pid=155, masterSystemTime=1730972311208 2024-11-07T09:38:31,315 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=e4cc6107007e595af5a0df7b5bc3711a, regionState=OPEN, openSeqNum=5, regionLocation=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:31,316 DEBUG [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:31,316 INFO [RS_OPEN_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:31,317 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=155, ppid=153, state=RUNNABLE, hasLock=false; OpenRegionProcedure e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 because future has completed 2024-11-07T09:38:31,320 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=153 2024-11-07T09:38:31,320 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=153, state=SUCCESS, hasLock=false; OpenRegionProcedure e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 in 266 msec 2024-11-07T09:38:31,322 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-07T09:38:31,323 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e4cc6107007e595af5a0df7b5bc3711a, REOPEN/MOVE in 616 msec 2024-11-07T09:38:31,326 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-11-07T09:38:31,326 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 623 msec 2024-11-07T09:38:31,329 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 808 msec 2024-11-07T09:38:31,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-11-07T09:38:31,337 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:38:31,352 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@604e524e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:31,352 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:38:31,352 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:38:31,354 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:38:31,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:38:31,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:38:31,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@496965d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:31,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:38:31,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:38:31,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:31,356 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34174, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:38:31,356 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70899504, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:31,374 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f36d46f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:31,374 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:38:31,374 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:38:31,374 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:38:31,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:38:31,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:38:31,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@290f202a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:31,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:38:31,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:38:31,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:31,376 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34194, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:38:31,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5784cfc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:31,377 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e5e61df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:31,377 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:38:31,377 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:38:31,378 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:38:31,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:38:31,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:38:31,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a7f0cee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:31,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:38:31,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:38:31,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:31,380 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34200, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:38:31,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@698bd018, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:31,385 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4885acdb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:31,385 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:38:31,385 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:38:31,386 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:38:31,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:38:31,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:38:31,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6417c85f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:31,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:38:31,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:38:31,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:31,388 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34230, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:38:31,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f4d5662, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:31,421 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cc99027, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:31,421 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:38:31,421 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:38:31,422 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:38:31,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:38:31,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:38:31,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a6bd449, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:31,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:38:31,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:38:31,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:31,429 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34240, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:38:31,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d3175c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:31,436 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3420b4f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:31,436 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:38:31,437 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:38:31,438 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:38:31,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:38:31,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:38:31,439 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@511caf3b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:31,439 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:38:31,439 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:38:31,439 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:31,440 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34260, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:38:31,441 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a774bb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:31,472 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d1b2f09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:31,472 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:38:31,472 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:38:31,474 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:38:31,474 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:38:31,474 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:38:31,475 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23633750, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:31,475 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:38:31,475 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:38:31,475 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:31,476 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34270, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:38:31,477 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@402611a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:31,576 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1102b71d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:31,576 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:38:31,576 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:38:31,579 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:38:31,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:38:31,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:38:31,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b730cc8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:31,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:38:31,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:38:31,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:31,582 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34302, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:38:31,583 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a5aaa09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:31,661 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22ff4806, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:31,661 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:38:31,661 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:38:31,662 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:38:31,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:38:31,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:38:31,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@367f0fd8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:31,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:38:31,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:38:31,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:31,665 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34308, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:38:31,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3164f44d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:31,668 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a8677bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:31,668 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9f1eb259a29b,41689,-1 for getting cluster id 2024-11-07T09:38:31,668 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T09:38:31,669 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039cc6dc-c845-4b6b-b77c-d5657b196eb1' 2024-11-07T09:38:31,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T09:38:31,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039cc6dc-c845-4b6b-b77c-d5657b196eb1" 2024-11-07T09:38:31,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ee9002f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:31,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9f1eb259a29b,41689,-1] 2024-11-07T09:38:31,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T09:38:31,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:31,671 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34330, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T09:38:31,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@183976c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T09:38:31,718 DEBUG [Thread-2499 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:38:31,720 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:38:31,720 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:38:31,721 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46550, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:38:31,723 DEBUG [Thread-2508 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:38:31,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5] 2024-11-07T09:38:31,724 DEBUG [Thread-2504 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:38:31,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:38:31,724 DEBUG [Thread-2493 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:38:31,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:38:31,728 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46566, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:38:31,732 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:38:31,733 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:38:31,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:38:31,734 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46578, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:38:31,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5] 2024-11-07T09:38:31,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5] 2024-11-07T09:38:31,781 DEBUG [Thread-2493 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:38:31,782 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:38:31,782 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e4cc6107007e595af5a0df7b5bc3711a 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T09:38:31,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:31,788 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46592, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:38:31,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5] 2024-11-07T09:38:31,803 DEBUG [Thread-2506 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:38:31,805 DEBUG [Thread-2506 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:38:31,805 DEBUG [Thread-2506 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:38:31,809 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46596, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:38:31,811 DEBUG [Thread-2506 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5] 2024-11-07T09:38:31,903 DEBUG [Thread-2497 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:38:31,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:38:31,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:38:31,905 DEBUG [Thread-2491 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:38:31,905 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46604, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:38:31,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:38:31,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:38:31,907 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46612, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:38:31,907 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5] 2024-11-07T09:38:31,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5] 2024-11-07T09:38:31,910 DEBUG [Thread-2510 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:38:31,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:38:31,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:38:31,912 DEBUG [Thread-2495 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:38:31,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:38:31,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:38:31,913 DEBUG [Thread-2502 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T09:38:31,913 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46630, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:38:31,914 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9f1eb259a29b,42895,1730972153970, seqNum=-1] 2024-11-07T09:38:31,914 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T09:38:31,915 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5] 2024-11-07T09:38:31,915 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46632, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:38:31,915 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46622, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T09:38:31,917 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5] 2024-11-07T09:38:31,920 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5] 2024-11-07T09:38:31,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:31,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 1 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1730972371918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:31,923 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:31,923 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:31,923 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:31,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:31,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46550 deadline: 1730972371919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:31,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:31,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:31,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:31,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:31,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46592 deadline: 1730972371920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:31,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:31,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:31,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:31,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:31,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46604 deadline: 1730972371921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:31,961 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:31,961 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:31,961 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:32,001 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T09:38:32,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-07T09:38:32,004 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T09:38:32,005 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T09:38:32,005 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T09:38:32,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-07T09:38:32,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:32,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46612 deadline: 1730972372006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:32,042 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:32,042 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:32,042 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:32,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:32,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46550 deadline: 1730972372036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:32,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:32,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:32,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:32,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:32,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46592 deadline: 1730972372036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:32,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:32,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:32,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:32,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:32,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1730972372040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:32,070 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:32,070 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:32,070 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:32,076 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411077ffc366721944c9aae56e2b3367cfeb1_e4cc6107007e595af5a0df7b5bc3711a is 50, key is test_row_0/A:col10/1730972311758/Put/seqid=0 2024-11-07T09:38:32,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:32,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46604 deadline: 1730972372063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:32,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:32,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:32,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:32,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-07T09:38:32,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742385_1561 (size=12154) 2024-11-07T09:38:32,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-07T09:38:32,166 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:32,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:32,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:32,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:32,168 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:32,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:32,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:32,170 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411077ffc366721944c9aae56e2b3367cfeb1_e4cc6107007e595af5a0df7b5bc3711a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411077ffc366721944c9aae56e2b3367cfeb1_e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:32,174 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/fd119cfe8c25426898eec077d4333d79, store: [table=TestAcidGuarantees family=A region=e4cc6107007e595af5a0df7b5bc3711a] 2024-11-07T09:38:32,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/fd119cfe8c25426898eec077d4333d79 is 175, key is test_row_0/A:col10/1730972311758/Put/seqid=0 2024-11-07T09:38:32,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:32,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46612 deadline: 1730972372153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:32,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:32,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:32,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:32,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:32,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46550 deadline: 1730972372273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:32,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:32,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:32,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:32,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:32,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46592 deadline: 1730972372273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:32,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:32,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:32,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:32,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:32,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1730972372291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:32,316 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:32,316 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:32,316 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:32,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:32,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46604 deadline: 1730972372293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:32,320 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:32,320 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:32,320 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:32,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742386_1562 (size=30955) 2024-11-07T09:38:32,329 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/fd119cfe8c25426898eec077d4333d79 2024-11-07T09:38:32,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-07T09:38:32,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-07T09:38:32,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:32,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:32,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:32,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:32,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:32,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:32,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/fc5577510e4647c0928888199b81a1ff is 50, key is test_row_0/B:col10/1730972311758/Put/seqid=0 2024-11-07T09:38:32,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:32,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46612 deadline: 1730972372444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:32,449 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:32,449 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:32,449 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:32,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742387_1563 (size=12001) 2024-11-07T09:38:32,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-07T09:38:32,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:32,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:32,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:32,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:32,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:32,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:32,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:32,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46550 deadline: 1730972372583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:32,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-07T09:38:32,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:32,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:32,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:32,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:32,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-07T09:38:32,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:32,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:32,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:32,698 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:32,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:32,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:32,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46592 deadline: 1730972372624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:32,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:32,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:32,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:32,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:32,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1730972372664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:32,737 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:32,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:32,737 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:32,737 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:32,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46604 deadline: 1730972372664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:32,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:32,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:32,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:32,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:32,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46612 deadline: 1730972372763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:32,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:32,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:32,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:32,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/fc5577510e4647c0928888199b81a1ff 2024-11-07T09:38:32,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-07T09:38:32,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:32,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:32,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:32,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:32,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:32,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:32,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/6acd9fea38984b73a9548e4e53c87daf is 50, key is test_row_0/C:col10/1730972311758/Put/seqid=0 2024-11-07T09:38:33,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742388_1564 (size=12001) 2024-11-07T09:38:33,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-07T09:38:33,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:33,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:33,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:33,062 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:33,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:33,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:33,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-07T09:38:33,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:33,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46550 deadline: 1730972373212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:33,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:33,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:33,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:33,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:33,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46592 deadline: 1730972373243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:33,266 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-07T09:38:33,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:33,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:33,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:33,266 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:33,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:33,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:33,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:33,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:33,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:33,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:33,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1730972373262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:33,283 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:33,283 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:33,283 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:33,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:33,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46604 deadline: 1730972373262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:33,284 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:33,284 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:33,284 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:33,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:33,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46612 deadline: 1730972373283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:33,341 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:33,341 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:33,341 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:33,410 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/6acd9fea38984b73a9548e4e53c87daf 2024-11-07T09:38:33,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/fd119cfe8c25426898eec077d4333d79 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/fd119cfe8c25426898eec077d4333d79 2024-11-07T09:38:33,424 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/fd119cfe8c25426898eec077d4333d79, entries=150, sequenceid=15, filesize=30.2 K 2024-11-07T09:38:33,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-07T09:38:33,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:33,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:33,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:33,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:33,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:33,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:33,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/fc5577510e4647c0928888199b81a1ff as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/fc5577510e4647c0928888199b81a1ff 2024-11-07T09:38:33,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/fc5577510e4647c0928888199b81a1ff, entries=150, sequenceid=15, filesize=11.7 K 2024-11-07T09:38:33,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/6acd9fea38984b73a9548e4e53c87daf as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/6acd9fea38984b73a9548e4e53c87daf 2024-11-07T09:38:33,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/6acd9fea38984b73a9548e4e53c87daf, entries=150, sequenceid=15, filesize=11.7 K 2024-11-07T09:38:33,513 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for e4cc6107007e595af5a0df7b5bc3711a in 1731ms, sequenceid=15, compaction requested=false 2024-11-07T09:38:33,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e4cc6107007e595af5a0df7b5bc3711a: 2024-11-07T09:38:33,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-07T09:38:33,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:33,594 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing e4cc6107007e595af5a0df7b5bc3711a 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-07T09:38:33,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110724dcd2198fa947a0aaa7aac282e7217d_e4cc6107007e595af5a0df7b5bc3711a is 50, key is test_row_0/A:col10/1730972311918/Put/seqid=0 2024-11-07T09:38:33,845 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T09:38:33,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742389_1565 (size=12154) 2024-11-07T09:38:33,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:33,908 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110724dcd2198fa947a0aaa7aac282e7217d_e4cc6107007e595af5a0df7b5bc3711a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110724dcd2198fa947a0aaa7aac282e7217d_e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:33,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/c1fec97221aa440ebc6232e2873c8a9b, store: [table=TestAcidGuarantees family=A region=e4cc6107007e595af5a0df7b5bc3711a] 2024-11-07T09:38:33,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/c1fec97221aa440ebc6232e2873c8a9b is 175, key is test_row_0/A:col10/1730972311918/Put/seqid=0 2024-11-07T09:38:34,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742390_1566 (size=30955) 2024-11-07T09:38:34,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-07T09:38:34,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:34,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:34,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:34,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46604 deadline: 1730972374372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:34,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:34,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46550 deadline: 1730972374377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:34,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:34,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:34,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:34,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:34,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46592 deadline: 1730972374380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:34,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:34,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:34,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:34,412 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:34,412 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:34,412 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:34,464 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/c1fec97221aa440ebc6232e2873c8a9b 2024-11-07T09:38:34,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:34,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46612 deadline: 1730972374412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:34,503 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:34,503 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:34,503 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:34,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:34,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1730972374503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:34,516 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:34,516 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:34,516 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:34,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/291a2f21ad134074b499724083d3ba1c is 50, key is test_row_0/B:col10/1730972311918/Put/seqid=0 2024-11-07T09:38:34,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:34,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46550 deadline: 1730972374513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:34,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:34,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46604 deadline: 1730972374523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:34,563 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:34,563 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:34,563 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:34,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:34,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:34,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:34,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:34,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46592 deadline: 1730972374568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:34,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:34,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:34,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:34,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:34,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46612 deadline: 1730972374613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:34,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:34,639 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:34,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1730972374623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:34,639 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:34,639 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:34,639 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:34,640 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:34,640 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:34,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742391_1567 (size=12001) 2024-11-07T09:38:34,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:34,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46604 deadline: 1730972374773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:34,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:34,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46550 deadline: 1730972374773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:34,779 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:34,779 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:34,779 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:34,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:34,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:34,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:34,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:34,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46592 deadline: 1730972374803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:34,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:34,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:34,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:34,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:34,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46612 deadline: 1730972374843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:34,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:34,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:34,901 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:34,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:34,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1730972374893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:34,905 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:34,905 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:34,905 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:35,061 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/291a2f21ad134074b499724083d3ba1c 2024-11-07T09:38:35,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/e4cabc2f6dae4d89b361766db0ac2c87 is 50, key is test_row_0/C:col10/1730972311918/Put/seqid=0 2024-11-07T09:38:35,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:35,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46604 deadline: 1730972375083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:35,104 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:35,104 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:35,104 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:35,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:35,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46550 deadline: 1730972375102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:35,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:35,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:35,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:35,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:35,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46592 deadline: 1730972375190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:35,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:35,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:35,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:35,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742392_1568 (size=12001) 2024-11-07T09:38:35,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:35,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46612 deadline: 1730972375212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:35,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:35,355 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:35,355 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:35,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1730972375213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:35,355 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:35,356 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:35,356 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:35,356 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:35,593 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/e4cabc2f6dae4d89b361766db0ac2c87 2024-11-07T09:38:35,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/c1fec97221aa440ebc6232e2873c8a9b as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/c1fec97221aa440ebc6232e2873c8a9b 2024-11-07T09:38:35,619 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/c1fec97221aa440ebc6232e2873c8a9b, entries=150, sequenceid=41, filesize=30.2 K 2024-11-07T09:38:35,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/291a2f21ad134074b499724083d3ba1c as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/291a2f21ad134074b499724083d3ba1c 2024-11-07T09:38:35,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:35,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46604 deadline: 1730972375613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:35,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:35,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:35,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:35,631 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/291a2f21ad134074b499724083d3ba1c, entries=150, sequenceid=41, filesize=11.7 K 2024-11-07T09:38:35,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/e4cabc2f6dae4d89b361766db0ac2c87 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/e4cabc2f6dae4d89b361766db0ac2c87 2024-11-07T09:38:35,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:35,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46550 deadline: 1730972375656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:35,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:35,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:35,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:35,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:35,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46592 deadline: 1730972375703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:35,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:35,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:35,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:35,714 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/e4cabc2f6dae4d89b361766db0ac2c87, entries=150, sequenceid=41, filesize=11.7 K 2024-11-07T09:38:35,715 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for e4cc6107007e595af5a0df7b5bc3711a in 2122ms, sequenceid=41, compaction requested=false 2024-11-07T09:38:35,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for e4cc6107007e595af5a0df7b5bc3711a: 2024-11-07T09:38:35,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:35,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-07T09:38:35,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-07T09:38:35,720 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-07T09:38:35,720 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.7120 sec 2024-11-07T09:38:35,723 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 3.7200 sec 2024-11-07T09:38:35,877 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:38:35,877 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e4cc6107007e595af5a0df7b5bc3711a 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-07T09:38:35,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:35,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411078c303b6ceca34e089e38c059f233dbb1_e4cc6107007e595af5a0df7b5bc3711a is 50, key is test_row_0/A:col10/1730972314336/Put/seqid=0 2024-11-07T09:38:36,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742393_1569 (size=14594) 2024-11-07T09:38:36,138 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:36,148 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411078c303b6ceca34e089e38c059f233dbb1_e4cc6107007e595af5a0df7b5bc3711a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411078c303b6ceca34e089e38c059f233dbb1_e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:36,149 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/c8a8315ed9e74feaa4574aee7c748a3a, store: [table=TestAcidGuarantees family=A region=e4cc6107007e595af5a0df7b5bc3711a] 2024-11-07T09:38:36,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/c8a8315ed9e74feaa4574aee7c748a3a is 175, key is test_row_0/A:col10/1730972314336/Put/seqid=0 2024-11-07T09:38:36,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-07T09:38:36,174 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:38:36,174 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T09:38:36,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-07T09:38:36,176 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T09:38:36,179 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T09:38:36,179 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T09:38:36,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-07T09:38:36,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742394_1570 (size=39549) 2024-11-07T09:38:36,256 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/c8a8315ed9e74feaa4574aee7c748a3a 2024-11-07T09:38:36,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/5b1b0ef1847149f9adafea9521a14093 is 50, key is test_row_0/B:col10/1730972314336/Put/seqid=0 2024-11-07T09:38:36,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-07T09:38:36,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-07T09:38:36,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:36,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:36,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:36,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:36,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:36,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:36,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742395_1571 (size=12001) 2024-11-07T09:38:36,444 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/5b1b0ef1847149f9adafea9521a14093 2024-11-07T09:38:36,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-07T09:38:36,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-07T09:38:36,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:36,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:36,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:36,503 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:36,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:36,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:36,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/c1ffa951dc574482b98aeac5f50d30f8 is 50, key is test_row_0/C:col10/1730972314336/Put/seqid=0 2024-11-07T09:38:36,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:36,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1730972376513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:36,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:36,527 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:36,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46612 deadline: 1730972376517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:36,527 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:36,528 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:36,528 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:36,528 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:36,528 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:36,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:36,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46612 deadline: 1730972376634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:36,642 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:36,643 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:36,643 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:36,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742396_1572 (size=12001) 2024-11-07T09:38:36,654 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/c1ffa951dc574482b98aeac5f50d30f8 2024-11-07T09:38:36,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/c8a8315ed9e74feaa4574aee7c748a3a as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/c8a8315ed9e74feaa4574aee7c748a3a 2024-11-07T09:38:36,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-07T09:38:36,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:36,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:36,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:36,662 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:36,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:36,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:36,665 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/c8a8315ed9e74feaa4574aee7c748a3a, entries=200, sequenceid=53, filesize=38.6 K 2024-11-07T09:38:36,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/5b1b0ef1847149f9adafea9521a14093 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/5b1b0ef1847149f9adafea9521a14093 2024-11-07T09:38:36,673 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/5b1b0ef1847149f9adafea9521a14093, entries=150, sequenceid=53, filesize=11.7 K 2024-11-07T09:38:36,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/c1ffa951dc574482b98aeac5f50d30f8 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/c1ffa951dc574482b98aeac5f50d30f8 2024-11-07T09:38:36,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/c1ffa951dc574482b98aeac5f50d30f8, entries=150, sequenceid=53, filesize=11.7 K 2024-11-07T09:38:36,699 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for e4cc6107007e595af5a0df7b5bc3711a in 822ms, sequenceid=53, compaction requested=true 2024-11-07T09:38:36,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e4cc6107007e595af5a0df7b5bc3711a: 2024-11-07T09:38:36,699 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:38:36,700 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:38:36,700 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): e4cc6107007e595af5a0df7b5bc3711a/A is initiating minor compaction (all files) 2024-11-07T09:38:36,700 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4cc6107007e595af5a0df7b5bc3711a/A in TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:36,700 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/fd119cfe8c25426898eec077d4333d79, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/c1fec97221aa440ebc6232e2873c8a9b, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/c8a8315ed9e74feaa4574aee7c748a3a] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp, totalSize=99.1 K 2024-11-07T09:38:36,700 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=65.38 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:36,700 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. files: [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/fd119cfe8c25426898eec077d4333d79, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/c1fec97221aa440ebc6232e2873c8a9b, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/c8a8315ed9e74feaa4574aee7c748a3a] 2024-11-07T09:38:36,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4cc6107007e595af5a0df7b5bc3711a:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T09:38:36,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:36,701 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd119cfe8c25426898eec077d4333d79, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1730972311758 2024-11-07T09:38:36,701 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1fec97221aa440ebc6232e2873c8a9b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1730972311910 2024-11-07T09:38:36,701 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8a8315ed9e74feaa4574aee7c748a3a, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1730972314336 2024-11-07T09:38:36,702 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:38:36,703 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:38:36,703 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): e4cc6107007e595af5a0df7b5bc3711a/B is initiating minor compaction (all files) 2024-11-07T09:38:36,703 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4cc6107007e595af5a0df7b5bc3711a/B in TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:36,703 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/fc5577510e4647c0928888199b81a1ff, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/291a2f21ad134074b499724083d3ba1c, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/5b1b0ef1847149f9adafea9521a14093] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp, totalSize=35.2 K 2024-11-07T09:38:36,704 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting fc5577510e4647c0928888199b81a1ff, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1730972311758 2024-11-07T09:38:36,704 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 291a2f21ad134074b499724083d3ba1c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1730972311910 2024-11-07T09:38:36,705 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b1b0ef1847149f9adafea9521a14093, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1730972314336 2024-11-07T09:38:36,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4cc6107007e595af5a0df7b5bc3711a:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T09:38:36,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:36,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4cc6107007e595af5a0df7b5bc3711a:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:38:36,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:38:36,749 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e4cc6107007e595af5a0df7b5bc3711a] 2024-11-07T09:38:36,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:36,759 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:38:36,759 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e4cc6107007e595af5a0df7b5bc3711a 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-07T09:38:36,790 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4cc6107007e595af5a0df7b5bc3711a#B#compaction#495 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 65.38 MB/second 2024-11-07T09:38:36,791 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/2b2ec2c512cd4fad9e5356a6a9598dc1 is 50, key is test_row_0/B:col10/1730972314336/Put/seqid=0 2024-11-07T09:38:36,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-07T09:38:36,808 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107af93e8b372554c92844fea63b95d54ee_e4cc6107007e595af5a0df7b5bc3711a store=[table=TestAcidGuarantees family=A region=e4cc6107007e595af5a0df7b5bc3711a] 2024-11-07T09:38:36,811 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107af93e8b372554c92844fea63b95d54ee_e4cc6107007e595af5a0df7b5bc3711a, store=[table=TestAcidGuarantees family=A region=e4cc6107007e595af5a0df7b5bc3711a] 2024-11-07T09:38:36,811 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107af93e8b372554c92844fea63b95d54ee_e4cc6107007e595af5a0df7b5bc3711a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e4cc6107007e595af5a0df7b5bc3711a] 2024-11-07T09:38:36,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-07T09:38:36,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:36,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:36,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:36,817 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:36,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:36,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:36,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:36,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1730972376810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:36,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:36,823 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:36,823 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:36,823 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:36,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46550 deadline: 1730972376818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:36,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:36,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:36,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:36,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:36,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46592 deadline: 1730972376820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:36,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:36,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:36,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:36,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742397_1573 (size=12104) 2024-11-07T09:38:36,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:36,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46604 deadline: 1730972376823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:36,829 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:36,829 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:36,830 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:36,868 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/2b2ec2c512cd4fad9e5356a6a9598dc1 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/2b2ec2c512cd4fad9e5356a6a9598dc1 2024-11-07T09:38:36,885 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e4cc6107007e595af5a0df7b5bc3711a/B of e4cc6107007e595af5a0df7b5bc3711a into 2b2ec2c512cd4fad9e5356a6a9598dc1(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:38:36,885 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4cc6107007e595af5a0df7b5bc3711a: 2024-11-07T09:38:36,885 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., storeName=e4cc6107007e595af5a0df7b5bc3711a/B, priority=13, startTime=1730972316702; duration=0sec 2024-11-07T09:38:36,885 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:38:36,885 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4cc6107007e595af5a0df7b5bc3711a:B 2024-11-07T09:38:36,885 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:38:36,886 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:38:36,886 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): e4cc6107007e595af5a0df7b5bc3711a/C is initiating minor compaction (all files) 2024-11-07T09:38:36,886 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4cc6107007e595af5a0df7b5bc3711a/C in TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:36,886 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/6acd9fea38984b73a9548e4e53c87daf, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/e4cabc2f6dae4d89b361766db0ac2c87, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/c1ffa951dc574482b98aeac5f50d30f8] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp, totalSize=35.2 K 2024-11-07T09:38:36,887 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 6acd9fea38984b73a9548e4e53c87daf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1730972311758 2024-11-07T09:38:36,887 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting e4cabc2f6dae4d89b361766db0ac2c87, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1730972311910 2024-11-07T09:38:36,887 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting c1ffa951dc574482b98aeac5f50d30f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1730972314336 2024-11-07T09:38:36,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411075681419ffe454b9d9901205dc908e875_e4cc6107007e595af5a0df7b5bc3711a is 50, key is test_row_0/A:col10/1730972316447/Put/seqid=0 2024-11-07T09:38:36,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:36,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46612 deadline: 1730972376857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:36,971 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:36,971 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:36,971 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:36,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-07T09:38:36,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:36,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:36,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:36,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:36,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:36,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:36,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:36,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1730972376934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:36,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:36,982 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:36,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46604 deadline: 1730972376935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:36,982 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:36,982 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:36,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:36,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46550 deadline: 1730972376935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:36,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:36,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:36,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:36,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:36,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:36,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:36,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:36,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46592 deadline: 1730972376935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:36,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:36,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:36,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:37,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-07T09:38:37,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:37,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:37,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:37,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:37,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:37,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:37,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742398_1574 (size=4469) 2024-11-07T09:38:37,178 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4cc6107007e595af5a0df7b5bc3711a#A#compaction#494 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 65.38 MB/second 2024-11-07T09:38:37,179 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/e4a4011bd6bd4b68a36748482538036b is 175, key is test_row_0/A:col10/1730972314336/Put/seqid=0 2024-11-07T09:38:37,191 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4cc6107007e595af5a0df7b5bc3711a#C#compaction#497 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 65.38 MB/second 2024-11-07T09:38:37,191 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/d417bdb1c21a4660aeadd751c7fece59 is 50, key is test_row_0/C:col10/1730972314336/Put/seqid=0 2024-11-07T09:38:37,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:37,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46604 deadline: 1730972377197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:37,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:37,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46550 deadline: 1730972377213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:37,229 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:37,229 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:37,229 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:37,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:37,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:37,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:37,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:37,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46592 deadline: 1730972377214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:37,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:37,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:37,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:37,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:37,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1730972377197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:37,273 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:37,273 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:37,273 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:37,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-07T09:38:37,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-07T09:38:37,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:37,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:37,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:37,341 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:37,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:37,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:37,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46612 deadline: 1730972377340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:37,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:37,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:37,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:37,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:37,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742399_1575 (size=17034) 2024-11-07T09:38:37,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742400_1576 (size=31058) 2024-11-07T09:38:37,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742401_1577 (size=12104) 2024-11-07T09:38:37,519 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/d417bdb1c21a4660aeadd751c7fece59 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/d417bdb1c21a4660aeadd751c7fece59 2024-11-07T09:38:37,524 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e4cc6107007e595af5a0df7b5bc3711a/C of e4cc6107007e595af5a0df7b5bc3711a into d417bdb1c21a4660aeadd751c7fece59(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:38:37,524 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4cc6107007e595af5a0df7b5bc3711a: 2024-11-07T09:38:37,524 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., storeName=e4cc6107007e595af5a0df7b5bc3711a/C, priority=13, startTime=1730972316709; duration=0sec 2024-11-07T09:38:37,524 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:37,524 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4cc6107007e595af5a0df7b5bc3711a:C 2024-11-07T09:38:37,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-07T09:38:37,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:37,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:37,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:37,536 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:37,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:37,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:37,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:37,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46604 deadline: 1730972377533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:37,612 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:37,612 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:37,612 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:37,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:37,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46550 deadline: 1730972377577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:37,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:37,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:37,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:37,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:37,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46592 deadline: 1730972377577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:37,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:37,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:37,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:37,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:37,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1730972377612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:37,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-07T09:38:37,766 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:37,766 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:37,766 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:37,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:37,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:37,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:37,767 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:37,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:37,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:37,784 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:37,788 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411075681419ffe454b9d9901205dc908e875_e4cc6107007e595af5a0df7b5bc3711a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411075681419ffe454b9d9901205dc908e875_e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:37,789 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/e4a4011bd6bd4b68a36748482538036b as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/e4a4011bd6bd4b68a36748482538036b 2024-11-07T09:38:37,790 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/f365ce54fe5e4cfcbb1858035e6f4a2a, store: [table=TestAcidGuarantees family=A region=e4cc6107007e595af5a0df7b5bc3711a] 2024-11-07T09:38:37,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/f365ce54fe5e4cfcbb1858035e6f4a2a is 175, key is test_row_0/A:col10/1730972316447/Put/seqid=0 2024-11-07T09:38:37,801 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e4cc6107007e595af5a0df7b5bc3711a/A of e4cc6107007e595af5a0df7b5bc3711a into e4a4011bd6bd4b68a36748482538036b(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 1sec to execute. 2024-11-07T09:38:37,801 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4cc6107007e595af5a0df7b5bc3711a: 2024-11-07T09:38:37,801 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., storeName=e4cc6107007e595af5a0df7b5bc3711a/A, priority=13, startTime=1730972316699; duration=1sec 2024-11-07T09:38:37,801 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:37,801 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4cc6107007e595af5a0df7b5bc3711a:A 2024-11-07T09:38:37,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:37,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46612 deadline: 1730972377853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:37,854 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:37,855 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:37,855 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:37,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742402_1578 (size=48139) 2024-11-07T09:38:37,906 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=79, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/f365ce54fe5e4cfcbb1858035e6f4a2a 2024-11-07T09:38:37,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-07T09:38:37,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:37,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:37,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:37,923 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:37,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:37,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:37,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/689605facccf4d61a10cb02b8f0387b0 is 50, key is test_row_0/B:col10/1730972316447/Put/seqid=0 2024-11-07T09:38:37,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742403_1579 (size=12001) 2024-11-07T09:38:37,981 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/689605facccf4d61a10cb02b8f0387b0 2024-11-07T09:38:38,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/2c78ced2ba9841bab8f0abf60b027135 is 50, key is test_row_0/C:col10/1730972316447/Put/seqid=0 2024-11-07T09:38:38,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742404_1580 (size=12001) 2024-11-07T09:38:38,044 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/2c78ced2ba9841bab8f0abf60b027135 2024-11-07T09:38:38,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/f365ce54fe5e4cfcbb1858035e6f4a2a as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/f365ce54fe5e4cfcbb1858035e6f4a2a 2024-11-07T09:38:38,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/f365ce54fe5e4cfcbb1858035e6f4a2a, entries=250, sequenceid=79, filesize=47.0 K 2024-11-07T09:38:38,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/689605facccf4d61a10cb02b8f0387b0 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/689605facccf4d61a10cb02b8f0387b0 2024-11-07T09:38:38,080 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/689605facccf4d61a10cb02b8f0387b0, entries=150, sequenceid=79, filesize=11.7 K 2024-11-07T09:38:38,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/2c78ced2ba9841bab8f0abf60b027135 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/2c78ced2ba9841bab8f0abf60b027135 2024-11-07T09:38:38,093 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/2c78ced2ba9841bab8f0abf60b027135, entries=150, sequenceid=79, filesize=11.7 K 2024-11-07T09:38:38,094 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for e4cc6107007e595af5a0df7b5bc3711a in 1335ms, sequenceid=79, compaction requested=false 2024-11-07T09:38:38,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e4cc6107007e595af5a0df7b5bc3711a: 2024-11-07T09:38:38,165 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:38:38,165 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e4cc6107007e595af5a0df7b5bc3711a 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T09:38:38,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:38,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-07T09:38:38,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:38,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:38,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:38,173 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:38,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:38,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:38,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107a21fd4b9fdd54beb86924f89edf7b2d9_e4cc6107007e595af5a0df7b5bc3711a is 50, key is test_row_0/A:col10/1730972316821/Put/seqid=0 2024-11-07T09:38:38,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-07T09:38:38,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:38,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:38,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:38,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:38,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:38,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:38,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-07T09:38:38,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742405_1581 (size=14594) 2024-11-07T09:38:38,497 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:38,507 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107a21fd4b9fdd54beb86924f89edf7b2d9_e4cc6107007e595af5a0df7b5bc3711a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107a21fd4b9fdd54beb86924f89edf7b2d9_e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:38,508 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/91d6a1d77f4c460fb1fb7e072cbc156e, store: [table=TestAcidGuarantees family=A region=e4cc6107007e595af5a0df7b5bc3711a] 2024-11-07T09:38:38,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/91d6a1d77f4c460fb1fb7e072cbc156e is 175, key is test_row_0/A:col10/1730972316821/Put/seqid=0 2024-11-07T09:38:38,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-07T09:38:38,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:38,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:38,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:38,537 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:38,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:38,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:38,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742406_1582 (size=39549) 2024-11-07T09:38:38,617 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=93, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/91d6a1d77f4c460fb1fb7e072cbc156e 2024-11-07T09:38:38,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/8d3747be76e64e1b99924cef6424d4bd is 50, key is test_row_0/B:col10/1730972316821/Put/seqid=0 2024-11-07T09:38:38,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-07T09:38:38,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:38,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:38,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:38,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:38,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:38,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:38,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:38,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46592 deadline: 1730972378693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:38,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:38,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:38,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:38,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:38,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1730972378702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:38,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:38,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46550 deadline: 1730972378735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:38,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:38,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:38,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:38,849 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:38,849 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:38,849 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:38,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-07T09:38:38,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:38,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:38,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:38,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:38,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:38,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:38,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742407_1583 (size=12001) 2024-11-07T09:38:38,885 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/8d3747be76e64e1b99924cef6424d4bd 2024-11-07T09:38:38,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:38,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46592 deadline: 1730972378842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:38,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:38,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:38,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:38,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:38,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46604 deadline: 1730972378850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:38,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:38,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:38,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:38,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:38,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46612 deadline: 1730972378905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:38,913 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 9f1eb259a29b,42895,1730972153970 for 'test_row_1' in e4cc6107007e595af5a0df7b5bc3711a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2396 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T09:38:38,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:38,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:38,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:38,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/3e38a4c9bb1f4a9f903252aa59e0320a is 50, key is test_row_0/C:col10/1730972316821/Put/seqid=0 2024-11-07T09:38:39,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:39,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46550 deadline: 1730972378932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:39,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:39,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:39,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:39,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:39,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1730972378953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:39,025 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:39,025 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:39,025 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:39,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:39,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46604 deadline: 1730972379025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:39,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:39,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:39,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:39,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742408_1584 (size=12001) 2024-11-07T09:38:39,035 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/3e38a4c9bb1f4a9f903252aa59e0320a 2024-11-07T09:38:39,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-07T09:38:39,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:39,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:39,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:39,065 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:39,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:39,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:39,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/91d6a1d77f4c460fb1fb7e072cbc156e as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/91d6a1d77f4c460fb1fb7e072cbc156e 2024-11-07T09:38:39,118 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/91d6a1d77f4c460fb1fb7e072cbc156e, entries=200, sequenceid=93, filesize=38.6 K 2024-11-07T09:38:39,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/8d3747be76e64e1b99924cef6424d4bd as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/8d3747be76e64e1b99924cef6424d4bd 2024-11-07T09:38:39,130 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/8d3747be76e64e1b99924cef6424d4bd, entries=150, sequenceid=93, filesize=11.7 K 2024-11-07T09:38:39,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/3e38a4c9bb1f4a9f903252aa59e0320a as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/3e38a4c9bb1f4a9f903252aa59e0320a 2024-11-07T09:38:39,135 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/3e38a4c9bb1f4a9f903252aa59e0320a, entries=150, sequenceid=93, filesize=11.7 K 2024-11-07T09:38:39,136 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for e4cc6107007e595af5a0df7b5bc3711a in 971ms, sequenceid=93, compaction requested=true 2024-11-07T09:38:39,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e4cc6107007e595af5a0df7b5bc3711a: 2024-11-07T09:38:39,136 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:38:39,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4cc6107007e595af5a0df7b5bc3711a:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T09:38:39,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:39,137 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:38:39,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4cc6107007e595af5a0df7b5bc3711a:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T09:38:39,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:39,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4cc6107007e595af5a0df7b5bc3711a:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:38:39,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:38:39,137 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118746 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:38:39,137 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): e4cc6107007e595af5a0df7b5bc3711a/A is initiating minor compaction (all files) 2024-11-07T09:38:39,137 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4cc6107007e595af5a0df7b5bc3711a/A in TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:39,138 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/e4a4011bd6bd4b68a36748482538036b, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/f365ce54fe5e4cfcbb1858035e6f4a2a, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/91d6a1d77f4c460fb1fb7e072cbc156e] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp, totalSize=116.0 K 2024-11-07T09:38:39,138 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=65.38 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:39,138 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. files: [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/e4a4011bd6bd4b68a36748482538036b, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/f365ce54fe5e4cfcbb1858035e6f4a2a, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/91d6a1d77f4c460fb1fb7e072cbc156e] 2024-11-07T09:38:39,138 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:38:39,139 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): e4cc6107007e595af5a0df7b5bc3711a/B is initiating minor compaction (all files) 2024-11-07T09:38:39,139 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4cc6107007e595af5a0df7b5bc3711a/B in TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:39,139 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/2b2ec2c512cd4fad9e5356a6a9598dc1, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/689605facccf4d61a10cb02b8f0387b0, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/8d3747be76e64e1b99924cef6424d4bd] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp, totalSize=35.3 K 2024-11-07T09:38:39,139 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4a4011bd6bd4b68a36748482538036b, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1730972314336 2024-11-07T09:38:39,139 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b2ec2c512cd4fad9e5356a6a9598dc1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1730972314336 2024-11-07T09:38:39,139 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting f365ce54fe5e4cfcbb1858035e6f4a2a, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1730972316362 2024-11-07T09:38:39,139 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 689605facccf4d61a10cb02b8f0387b0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1730972316447 2024-11-07T09:38:39,140 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91d6a1d77f4c460fb1fb7e072cbc156e, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1730972316806 2024-11-07T09:38:39,140 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d3747be76e64e1b99924cef6424d4bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1730972316806 2024-11-07T09:38:39,176 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:38:39,176 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e4cc6107007e595af5a0df7b5bc3711a 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-07T09:38:39,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:39,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-07T09:38:39,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:39,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:39,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:39,222 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:39,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:39,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:39,231 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e4cc6107007e595af5a0df7b5bc3711a] 2024-11-07T09:38:39,243 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4cc6107007e595af5a0df7b5bc3711a#B#compaction#504 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 65.38 MB/second 2024-11-07T09:38:39,243 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/2bcf8afb2a924e66b9eaa6599f125a69 is 50, key is test_row_0/B:col10/1730972316821/Put/seqid=0 2024-11-07T09:38:39,310 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411073ffcc21df959448b84aa2ca15cdbc574_e4cc6107007e595af5a0df7b5bc3711a store=[table=TestAcidGuarantees family=A region=e4cc6107007e595af5a0df7b5bc3711a] 2024-11-07T09:38:39,312 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411073ffcc21df959448b84aa2ca15cdbc574_e4cc6107007e595af5a0df7b5bc3711a, store=[table=TestAcidGuarantees family=A region=e4cc6107007e595af5a0df7b5bc3711a] 2024-11-07T09:38:39,312 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411073ffcc21df959448b84aa2ca15cdbc574_e4cc6107007e595af5a0df7b5bc3711a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e4cc6107007e595af5a0df7b5bc3711a] 2024-11-07T09:38:39,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:39,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46604 deadline: 1730972379313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:39,418 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:39,418 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:39,418 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:39,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-07T09:38:39,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:39,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:39,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:39,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:39,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:39,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:39,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:39,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46592 deadline: 1730972379417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:39,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:39,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:39,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:39,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:39,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46550 deadline: 1730972379418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:39,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:39,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:39,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:39,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:39,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1730972379423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:39,533 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:39,533 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:39,533 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:39,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107e95db33f3b814e99a99b52e9aab077f4_e4cc6107007e595af5a0df7b5bc3711a is 50, key is test_row_0/A:col10/1730972318724/Put/seqid=0 2024-11-07T09:38:39,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-07T09:38:39,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:39,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:39,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:39,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:39,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:39,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:39,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46604 deadline: 1730972379532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:39,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:39,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:39,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:39,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:39,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:39,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46592 deadline: 1730972379588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:39,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:39,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:39,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:39,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742409_1585 (size=12207) 2024-11-07T09:38:39,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:39,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46550 deadline: 1730972379604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:39,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:39,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1730972379645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:39,680 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:39,680 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:39,680 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:39,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:39,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:39,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:39,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-07T09:38:39,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:39,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:39,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:39,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:39,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:39,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:39,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:39,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46604 deadline: 1730972379795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:39,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:39,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46592 deadline: 1730972379809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:39,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:39,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:39,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:39,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742410_1586 (size=4469) 2024-11-07T09:38:39,834 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4cc6107007e595af5a0df7b5bc3711a#A#compaction#503 average throughput is 0.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 65.38 MB/second 2024-11-07T09:38:39,834 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/e65e7816006b46db8ad331ba51db7759 is 175, key is test_row_0/A:col10/1730972316821/Put/seqid=0 2024-11-07T09:38:39,860 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:39,860 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:39,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:39,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:39,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1730972379893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:39,919 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:39,919 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:39,919 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:39,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742411_1587 (size=14594) 2024-11-07T09:38:39,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:39,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46550 deadline: 1730972379923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:39,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:39,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:39,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:39,981 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-07T09:38:39,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:39,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:39,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:39,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:39,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:39,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:40,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742412_1588 (size=31161) 2024-11-07T09:38:40,042 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/e65e7816006b46db8ad331ba51db7759 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/e65e7816006b46db8ad331ba51db7759 2024-11-07T09:38:40,060 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/2bcf8afb2a924e66b9eaa6599f125a69 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/2bcf8afb2a924e66b9eaa6599f125a69 2024-11-07T09:38:40,134 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e4cc6107007e595af5a0df7b5bc3711a/A of e4cc6107007e595af5a0df7b5bc3711a into e65e7816006b46db8ad331ba51db7759(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:38:40,134 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4cc6107007e595af5a0df7b5bc3711a: 2024-11-07T09:38:40,134 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., storeName=e4cc6107007e595af5a0df7b5bc3711a/A, priority=13, startTime=1730972319136; duration=0sec 2024-11-07T09:38:40,134 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:38:40,134 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4cc6107007e595af5a0df7b5bc3711a:A 2024-11-07T09:38:40,135 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:38:40,135 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e4cc6107007e595af5a0df7b5bc3711a/B of e4cc6107007e595af5a0df7b5bc3711a into 2bcf8afb2a924e66b9eaa6599f125a69(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:38:40,135 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4cc6107007e595af5a0df7b5bc3711a: 2024-11-07T09:38:40,135 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., storeName=e4cc6107007e595af5a0df7b5bc3711a/B, priority=13, startTime=1730972319136; duration=0sec 2024-11-07T09:38:40,135 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:40,135 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4cc6107007e595af5a0df7b5bc3711a:B 2024-11-07T09:38:40,136 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:38:40,136 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): e4cc6107007e595af5a0df7b5bc3711a/C is initiating minor compaction (all files) 2024-11-07T09:38:40,136 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4cc6107007e595af5a0df7b5bc3711a/C in TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:40,136 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/d417bdb1c21a4660aeadd751c7fece59, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/2c78ced2ba9841bab8f0abf60b027135, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/3e38a4c9bb1f4a9f903252aa59e0320a] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp, totalSize=35.3 K 2024-11-07T09:38:40,137 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting d417bdb1c21a4660aeadd751c7fece59, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1730972314336 2024-11-07T09:38:40,137 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c78ced2ba9841bab8f0abf60b027135, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1730972316447 2024-11-07T09:38:40,137 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e38a4c9bb1f4a9f903252aa59e0320a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1730972316806 2024-11-07T09:38:40,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-07T09:38:40,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:40,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:40,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:40,150 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:40,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:40,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:40,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:40,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46592 deadline: 1730972380148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:40,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:40,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:40,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:40,190 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4cc6107007e595af5a0df7b5bc3711a#C#compaction#506 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 65.38 MB/second 2024-11-07T09:38:40,191 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/a2b1271857f143acb62c54f48be29199 is 50, key is test_row_0/C:col10/1730972316821/Put/seqid=0 2024-11-07T09:38:40,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:40,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46604 deadline: 1730972380165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:40,211 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:40,211 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:40,211 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:40,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:40,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1730972380233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:40,243 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:40,243 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:40,243 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:40,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:40,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46550 deadline: 1730972380283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:40,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:40,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:40,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:40,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-07T09:38:40,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:40,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:40,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:40,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:40,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:40,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:40,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-07T09:38:40,360 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:40,364 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107e95db33f3b814e99a99b52e9aab077f4_e4cc6107007e595af5a0df7b5bc3711a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107e95db33f3b814e99a99b52e9aab077f4_e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:40,365 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/338b501de47e471b87678cf61758c18d, store: [table=TestAcidGuarantees family=A region=e4cc6107007e595af5a0df7b5bc3711a] 2024-11-07T09:38:40,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/338b501de47e471b87678cf61758c18d is 175, key is test_row_0/A:col10/1730972318724/Put/seqid=0 2024-11-07T09:38:40,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-07T09:38:40,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:40,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:40,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:40,491 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:40,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:40,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:40,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742414_1590 (size=39549) 2024-11-07T09:38:40,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742413_1589 (size=12207) 2024-11-07T09:38:40,576 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/a2b1271857f143acb62c54f48be29199 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/a2b1271857f143acb62c54f48be29199 2024-11-07T09:38:40,591 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e4cc6107007e595af5a0df7b5bc3711a/C of e4cc6107007e595af5a0df7b5bc3711a into a2b1271857f143acb62c54f48be29199(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:38:40,591 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4cc6107007e595af5a0df7b5bc3711a: 2024-11-07T09:38:40,591 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., storeName=e4cc6107007e595af5a0df7b5bc3711a/C, priority=13, startTime=1730972319137; duration=0sec 2024-11-07T09:38:40,591 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:40,591 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4cc6107007e595af5a0df7b5bc3711a:C 2024-11-07T09:38:40,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-07T09:38:40,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:40,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:40,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:40,658 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:40,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:40,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:40,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:40,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46592 deadline: 1730972380693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:40,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:40,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:40,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:40,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:40,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46604 deadline: 1730972380753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:40,763 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:40,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:40,763 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:40,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:40,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1730972380756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:40,764 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:40,764 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:40,764 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:40,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-07T09:38:40,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:40,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:40,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:40,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:40,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:40,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:40,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:40,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46550 deadline: 1730972380841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:40,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:40,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:40,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:40,966 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=120, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/338b501de47e471b87678cf61758c18d 2024-11-07T09:38:40,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:40,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46612 deadline: 1730972380935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:40,968 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 9f1eb259a29b,42895,1730972153970 for 'test_row_1' in e4cc6107007e595af5a0df7b5bc3711a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4450 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T09:38:40,968 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:40,968 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:40,968 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:41,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-07T09:38:41,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:41,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:41,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:41,035 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:41,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:41,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:41,053 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/7e2c6397ba104b2fb4c6819a1851071f is 50, key is test_row_0/B:col10/1730972318724/Put/seqid=0 2024-11-07T09:38:41,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742415_1591 (size=12001) 2024-11-07T09:38:41,139 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/7e2c6397ba104b2fb4c6819a1851071f 2024-11-07T09:38:41,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/85546a63f8034046975fde0847299591 is 50, key is test_row_0/C:col10/1730972318724/Put/seqid=0 2024-11-07T09:38:41,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-07T09:38:41,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:41,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:41,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:41,188 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:41,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:41,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:41,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742416_1592 (size=12001) 2024-11-07T09:38:41,330 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/85546a63f8034046975fde0847299591 2024-11-07T09:38:41,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/338b501de47e471b87678cf61758c18d as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/338b501de47e471b87678cf61758c18d 2024-11-07T09:38:41,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/338b501de47e471b87678cf61758c18d, entries=200, sequenceid=120, filesize=38.6 K 2024-11-07T09:38:41,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/7e2c6397ba104b2fb4c6819a1851071f as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/7e2c6397ba104b2fb4c6819a1851071f 2024-11-07T09:38:41,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/7e2c6397ba104b2fb4c6819a1851071f, entries=150, sequenceid=120, filesize=11.7 K 2024-11-07T09:38:41,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/85546a63f8034046975fde0847299591 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/85546a63f8034046975fde0847299591 2024-11-07T09:38:41,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/85546a63f8034046975fde0847299591, entries=150, sequenceid=120, filesize=11.7 K 2024-11-07T09:38:41,347 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for e4cc6107007e595af5a0df7b5bc3711a in 2170ms, sequenceid=120, compaction requested=false 2024-11-07T09:38:41,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e4cc6107007e595af5a0df7b5bc3711a: 2024-11-07T09:38:41,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-07T09:38:41,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:41,362 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing e4cc6107007e595af5a0df7b5bc3711a 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-07T09:38:41,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110757cf1ea4cc6f4876845515006bb5837a_e4cc6107007e595af5a0df7b5bc3711a is 50, key is test_row_0/A:col10/1730972319417/Put/seqid=0 2024-11-07T09:38:41,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:41,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:41,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742417_1593 (size=12204) 2024-11-07T09:38:41,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:41,786 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110757cf1ea4cc6f4876845515006bb5837a_e4cc6107007e595af5a0df7b5bc3711a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110757cf1ea4cc6f4876845515006bb5837a_e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:41,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/64ddd7511f1b4cf2b4a70860709208c3, store: [table=TestAcidGuarantees family=A region=e4cc6107007e595af5a0df7b5bc3711a] 2024-11-07T09:38:41,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/64ddd7511f1b4cf2b4a70860709208c3 is 175, key is test_row_0/A:col10/1730972319417/Put/seqid=0 2024-11-07T09:38:41,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742418_1594 (size=31005) 2024-11-07T09:38:41,894 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/64ddd7511f1b4cf2b4a70860709208c3 2024-11-07T09:38:41,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/245209913f4342c482edd81a18001e21 is 50, key is test_row_0/B:col10/1730972319417/Put/seqid=0 2024-11-07T09:38:42,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742419_1595 (size=12051) 2024-11-07T09:38:42,180 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/245209913f4342c482edd81a18001e21 2024-11-07T09:38:42,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/0d11bda8fdeb4c3c9e5a4a18df5ca7c4 is 50, key is test_row_0/C:col10/1730972319417/Put/seqid=0 2024-11-07T09:38:42,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:42,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46592 deadline: 1730972382201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:42,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:42,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:42,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:42,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:42,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1730972382259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:42,279 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:42,279 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:42,279 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:42,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:42,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46604 deadline: 1730972382262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:42,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:42,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:42,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:42,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:42,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46550 deadline: 1730972382272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:42,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742420_1596 (size=12051) 2024-11-07T09:38:42,310 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/0d11bda8fdeb4c3c9e5a4a18df5ca7c4 2024-11-07T09:38:42,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/64ddd7511f1b4cf2b4a70860709208c3 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/64ddd7511f1b4cf2b4a70860709208c3 2024-11-07T09:38:42,318 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/64ddd7511f1b4cf2b4a70860709208c3, entries=150, sequenceid=133, filesize=30.3 K 2024-11-07T09:38:42,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/245209913f4342c482edd81a18001e21 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/245209913f4342c482edd81a18001e21 2024-11-07T09:38:42,323 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/245209913f4342c482edd81a18001e21, entries=150, sequenceid=133, filesize=11.8 K 2024-11-07T09:38:42,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/0d11bda8fdeb4c3c9e5a4a18df5ca7c4 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/0d11bda8fdeb4c3c9e5a4a18df5ca7c4 2024-11-07T09:38:42,327 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/0d11bda8fdeb4c3c9e5a4a18df5ca7c4, entries=150, sequenceid=133, filesize=11.8 K 2024-11-07T09:38:42,333 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=161.02 KB/164880 for e4cc6107007e595af5a0df7b5bc3711a in 970ms, sequenceid=133, compaction requested=true 2024-11-07T09:38:42,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for e4cc6107007e595af5a0df7b5bc3711a: 2024-11-07T09:38:42,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:42,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-07T09:38:42,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-07T09:38:42,339 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 6.1640 sec 2024-11-07T09:38:42,339 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-07T09:38:42,339 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 6.1550 sec 2024-11-07T09:38:42,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:42,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:42,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:42,453 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:38:42,453 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e4cc6107007e595af5a0df7b5bc3711a 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-07T09:38:42,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:42,469 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107c5e2f4891cea4582a955cc197d021dbb_e4cc6107007e595af5a0df7b5bc3711a is 50, key is test_row_0/A:col10/1730972322270/Put/seqid=0 2024-11-07T09:38:42,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:42,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46604 deadline: 1730972382469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:42,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:42,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:42,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:42,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46550 deadline: 1730972382469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:42,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:42,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:42,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:42,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:42,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:42,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46592 deadline: 1730972382470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:42,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:42,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:42,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:42,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:42,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1730972382472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:42,512 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:42,512 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:42,512 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:42,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:42,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46550 deadline: 1730972382613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:42,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:42,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:42,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:42,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:42,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46592 deadline: 1730972382614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:42,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:42,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:42,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46604 deadline: 1730972382625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:42,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:42,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:42,742 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:42,742 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:42,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742421_1597 (size=14794) 2024-11-07T09:38:42,742 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:42,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:42,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1730972382625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:42,743 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:42,744 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:42,744 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:42,744 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:42,748 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107c5e2f4891cea4582a955cc197d021dbb_e4cc6107007e595af5a0df7b5bc3711a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107c5e2f4891cea4582a955cc197d021dbb_e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:42,750 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/f327137223704eb49e86df06f666903d, store: [table=TestAcidGuarantees family=A region=e4cc6107007e595af5a0df7b5bc3711a] 2024-11-07T09:38:42,751 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/f327137223704eb49e86df06f666903d is 175, key is test_row_0/A:col10/1730972322270/Put/seqid=0 2024-11-07T09:38:42,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742422_1598 (size=39749) 2024-11-07T09:38:42,859 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=161, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/f327137223704eb49e86df06f666903d 2024-11-07T09:38:42,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:42,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46550 deadline: 1730972382863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:42,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:42,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:42,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:42,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:42,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46604 deadline: 1730972382980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:42,998 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:42,999 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:42,999 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:42,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:42,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1730972382980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:43,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:43,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:43,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:43,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:43,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46592 deadline: 1730972382994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:43,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:43,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:43,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:43,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/9058eef2ede4427da2c4de91bdac369c is 50, key is test_row_0/B:col10/1730972322270/Put/seqid=0 2024-11-07T09:38:43,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:43,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46550 deadline: 1730972383183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:43,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:43,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:43,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:43,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742423_1599 (size=12151) 2024-11-07T09:38:43,327 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/9058eef2ede4427da2c4de91bdac369c 2024-11-07T09:38:43,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:43,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46604 deadline: 1730972383302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:43,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:43,337 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:43,337 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:43,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1730972383303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:43,337 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:43,339 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:43,339 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:43,339 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:43,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:43,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46592 deadline: 1730972383323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:43,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:43,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:43,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:43,351 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/c85ac8583e414a9784ae399a5f441d9a is 50, key is test_row_0/C:col10/1730972322270/Put/seqid=0 2024-11-07T09:38:43,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742424_1600 (size=12151) 2024-11-07T09:38:43,529 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/c85ac8583e414a9784ae399a5f441d9a 2024-11-07T09:38:43,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/f327137223704eb49e86df06f666903d as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/f327137223704eb49e86df06f666903d 2024-11-07T09:38:43,567 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/f327137223704eb49e86df06f666903d, entries=200, sequenceid=161, filesize=38.8 K 2024-11-07T09:38:43,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/9058eef2ede4427da2c4de91bdac369c as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/9058eef2ede4427da2c4de91bdac369c 2024-11-07T09:38:43,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/9058eef2ede4427da2c4de91bdac369c, entries=150, sequenceid=161, filesize=11.9 K 2024-11-07T09:38:43,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/c85ac8583e414a9784ae399a5f441d9a as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/c85ac8583e414a9784ae399a5f441d9a 2024-11-07T09:38:43,598 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/c85ac8583e414a9784ae399a5f441d9a, entries=150, sequenceid=161, filesize=11.9 K 2024-11-07T09:38:43,599 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for e4cc6107007e595af5a0df7b5bc3711a in 1146ms, sequenceid=161, compaction requested=true 2024-11-07T09:38:43,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e4cc6107007e595af5a0df7b5bc3711a: 2024-11-07T09:38:43,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4cc6107007e595af5a0df7b5bc3711a:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T09:38:43,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:43,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4cc6107007e595af5a0df7b5bc3711a:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T09:38:43,600 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T09:38:43,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:43,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4cc6107007e595af5a0df7b5bc3711a:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:38:43,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:38:43,600 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T09:38:43,601 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141464 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T09:38:43,601 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): e4cc6107007e595af5a0df7b5bc3711a/A is initiating minor compaction (all files) 2024-11-07T09:38:43,601 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4cc6107007e595af5a0df7b5bc3711a/A in TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:43,601 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/e65e7816006b46db8ad331ba51db7759, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/338b501de47e471b87678cf61758c18d, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/64ddd7511f1b4cf2b4a70860709208c3, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/f327137223704eb49e86df06f666903d] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp, totalSize=138.1 K 2024-11-07T09:38:43,601 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=65.38 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:43,602 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. files: [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/e65e7816006b46db8ad331ba51db7759, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/338b501de47e471b87678cf61758c18d, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/64ddd7511f1b4cf2b4a70860709208c3, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/f327137223704eb49e86df06f666903d] 2024-11-07T09:38:43,602 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T09:38:43,602 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): e4cc6107007e595af5a0df7b5bc3711a/B is initiating minor compaction (all files) 2024-11-07T09:38:43,602 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4cc6107007e595af5a0df7b5bc3711a/B in TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:43,602 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/2bcf8afb2a924e66b9eaa6599f125a69, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/7e2c6397ba104b2fb4c6819a1851071f, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/245209913f4342c482edd81a18001e21, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/9058eef2ede4427da2c4de91bdac369c] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp, totalSize=47.3 K 2024-11-07T09:38:43,602 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting e65e7816006b46db8ad331ba51db7759, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1730972316806 2024-11-07T09:38:43,602 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bcf8afb2a924e66b9eaa6599f125a69, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1730972316806 2024-11-07T09:38:43,603 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e2c6397ba104b2fb4c6819a1851071f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1730972318693 2024-11-07T09:38:43,603 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 338b501de47e471b87678cf61758c18d, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1730972318541 2024-11-07T09:38:43,603 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 245209913f4342c482edd81a18001e21, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1730972319333 2024-11-07T09:38:43,604 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64ddd7511f1b4cf2b4a70860709208c3, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1730972319333 2024-11-07T09:38:43,604 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 9058eef2ede4427da2c4de91bdac369c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1730972322080 2024-11-07T09:38:43,604 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting f327137223704eb49e86df06f666903d, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1730972322006 2024-11-07T09:38:43,618 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4cc6107007e595af5a0df7b5bc3711a#B#compaction#515 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 65.38 MB/second 2024-11-07T09:38:43,618 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/ef0910e7bd9840838d641dbd982c1111 is 50, key is test_row_0/B:col10/1730972322270/Put/seqid=0 2024-11-07T09:38:43,665 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e4cc6107007e595af5a0df7b5bc3711a] 2024-11-07T09:38:43,866 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107f4160b8a02fa4ccb830cc12a4149ca1d_e4cc6107007e595af5a0df7b5bc3711a store=[table=TestAcidGuarantees family=A region=e4cc6107007e595af5a0df7b5bc3711a] 2024-11-07T09:38:43,946 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107f4160b8a02fa4ccb830cc12a4149ca1d_e4cc6107007e595af5a0df7b5bc3711a, store=[table=TestAcidGuarantees family=A region=e4cc6107007e595af5a0df7b5bc3711a] 2024-11-07T09:38:43,946 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107f4160b8a02fa4ccb830cc12a4149ca1d_e4cc6107007e595af5a0df7b5bc3711a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e4cc6107007e595af5a0df7b5bc3711a] 2024-11-07T09:38:43,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:43,950 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:38:43,950 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e4cc6107007e595af5a0df7b5bc3711a 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T09:38:44,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411073b0853898dce4277b033319f6e050573_e4cc6107007e595af5a0df7b5bc3711a is 50, key is test_row_0/A:col10/1730972323949/Put/seqid=0 2024-11-07T09:38:44,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742426_1602 (size=4469) 2024-11-07T09:38:44,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742425_1601 (size=12493) 2024-11-07T09:38:44,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:44,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1730972384195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:44,216 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:44,216 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:44,216 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:44,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:44,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46604 deadline: 1730972384200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:44,219 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:44,219 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:44,219 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:44,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:44,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46550 deadline: 1730972384202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:44,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:44,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:44,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:44,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:44,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46592 deadline: 1730972384203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:44,230 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:44,230 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:44,230 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:44,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742427_1603 (size=12304) 2024-11-07T09:38:44,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-07T09:38:44,364 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:38:44,365 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T09:38:44,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-07T09:38:44,368 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T09:38:44,369 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T09:38:44,369 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T09:38:44,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-07T09:38:44,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:44,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1730972384323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:44,411 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:44,411 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:44,411 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:44,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:44,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46604 deadline: 1730972384324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:44,412 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:44,412 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:44,412 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:44,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:44,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46550 deadline: 1730972384324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:44,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:44,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:44,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:44,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:44,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46592 deadline: 1730972384332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:44,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:44,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:44,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:44,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-07T09:38:44,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T09:38:44,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:44,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:44,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:44,522 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:44,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:44,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:44,568 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4cc6107007e595af5a0df7b5bc3711a#A#compaction#516 average throughput is 0.03 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 65.38 MB/second 2024-11-07T09:38:44,568 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/6d14692d71254e929fc7bdee946f952a is 175, key is test_row_0/A:col10/1730972322270/Put/seqid=0 2024-11-07T09:38:44,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:44,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1730972384614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:44,669 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:44,669 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:44,669 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:44,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:44,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46604 deadline: 1730972384624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:44,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:44,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:44,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:44,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:44,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46550 deadline: 1730972384624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:44,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:44,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:44,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:44,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:44,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46592 deadline: 1730972384624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:44,676 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/ef0910e7bd9840838d641dbd982c1111 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/ef0910e7bd9840838d641dbd982c1111 2024-11-07T09:38:44,682 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e4cc6107007e595af5a0df7b5bc3711a/B of e4cc6107007e595af5a0df7b5bc3711a into ef0910e7bd9840838d641dbd982c1111(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 1sec to execute. 2024-11-07T09:38:44,682 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4cc6107007e595af5a0df7b5bc3711a: 2024-11-07T09:38:44,682 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., storeName=e4cc6107007e595af5a0df7b5bc3711a/B, priority=12, startTime=1730972323600; duration=1sec 2024-11-07T09:38:44,682 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:38:44,682 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4cc6107007e595af5a0df7b5bc3711a:B 2024-11-07T09:38:44,682 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T09:38:44,683 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:44,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-07T09:38:44,685 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T09:38:44,685 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): e4cc6107007e595af5a0df7b5bc3711a/C is initiating minor compaction (all files) 2024-11-07T09:38:44,685 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4cc6107007e595af5a0df7b5bc3711a/C in TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:44,685 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/a2b1271857f143acb62c54f48be29199, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/85546a63f8034046975fde0847299591, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/0d11bda8fdeb4c3c9e5a4a18df5ca7c4, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/c85ac8583e414a9784ae399a5f441d9a] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp, totalSize=47.3 K 2024-11-07T09:38:44,686 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting a2b1271857f143acb62c54f48be29199, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1730972316806 2024-11-07T09:38:44,687 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 85546a63f8034046975fde0847299591, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1730972318693 2024-11-07T09:38:44,687 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411073b0853898dce4277b033319f6e050573_e4cc6107007e595af5a0df7b5bc3711a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411073b0853898dce4277b033319f6e050573_e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:44,689 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d11bda8fdeb4c3c9e5a4a18df5ca7c4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1730972319333 2024-11-07T09:38:44,689 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting c85ac8583e414a9784ae399a5f441d9a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1730972322080 2024-11-07T09:38:44,690 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/9f11bbb34a9e4565a95ea7328e93da11, store: [table=TestAcidGuarantees family=A region=e4cc6107007e595af5a0df7b5bc3711a] 2024-11-07T09:38:44,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/9f11bbb34a9e4565a95ea7328e93da11 is 175, key is test_row_0/A:col10/1730972323949/Put/seqid=0 2024-11-07T09:38:44,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:44,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:44,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:44,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T09:38:44,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:44,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:44,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:44,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:44,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:44,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:44,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742428_1604 (size=31447) 2024-11-07T09:38:44,755 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4cc6107007e595af5a0df7b5bc3711a#C#compaction#518 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 65.38 MB/second 2024-11-07T09:38:44,755 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/92ece59a5a3745c79b39542e651799ed is 50, key is test_row_0/C:col10/1730972322270/Put/seqid=0 2024-11-07T09:38:44,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742429_1605 (size=31105) 2024-11-07T09:38:44,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742430_1606 (size=12493) 2024-11-07T09:38:44,809 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/92ece59a5a3745c79b39542e651799ed as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/92ece59a5a3745c79b39542e651799ed 2024-11-07T09:38:44,821 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e4cc6107007e595af5a0df7b5bc3711a/C of e4cc6107007e595af5a0df7b5bc3711a into 92ece59a5a3745c79b39542e651799ed(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:38:44,821 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4cc6107007e595af5a0df7b5bc3711a: 2024-11-07T09:38:44,821 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., storeName=e4cc6107007e595af5a0df7b5bc3711a/C, priority=12, startTime=1730972323600; duration=0sec 2024-11-07T09:38:44,821 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:44,821 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4cc6107007e595af5a0df7b5bc3711a:C 2024-11-07T09:38:44,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T09:38:44,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:44,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:44,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:44,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:44,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:44,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:44,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:44,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1730972384991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:44,993 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:44,993 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:44,993 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:44,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:44,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46604 deadline: 1730972384991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:44,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-07T09:38:44,994 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:44,994 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:44,994 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:45,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:45,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46612 deadline: 1730972385004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:45,055 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T09:38:45,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:45,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:45,068 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 9f1eb259a29b,42895,1730972153970 for 'test_row_1' in e4cc6107007e595af5a0df7b5bc3711a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8551 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T09:38:45,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:45,069 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:45,069 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:45,069 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:45,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:45,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:45,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:45,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:45,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46550 deadline: 1730972385004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:45,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:45,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:45,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:45,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:45,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46592 deadline: 1730972385053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:45,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:45,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:45,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:45,155 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/6d14692d71254e929fc7bdee946f952a as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/6d14692d71254e929fc7bdee946f952a 2024-11-07T09:38:45,160 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e4cc6107007e595af5a0df7b5bc3711a/A of e4cc6107007e595af5a0df7b5bc3711a into 6d14692d71254e929fc7bdee946f952a(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 1sec to execute. 2024-11-07T09:38:45,160 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4cc6107007e595af5a0df7b5bc3711a: 2024-11-07T09:38:45,160 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., storeName=e4cc6107007e595af5a0df7b5bc3711a/A, priority=12, startTime=1730972323600; duration=1sec 2024-11-07T09:38:45,160 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:45,160 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4cc6107007e595af5a0df7b5bc3711a:A 2024-11-07T09:38:45,174 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/9f11bbb34a9e4565a95ea7328e93da11 2024-11-07T09:38:45,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/04bd274447bb490c937f47c810c4d086 is 50, key is test_row_0/B:col10/1730972323949/Put/seqid=0 2024-11-07T09:38:45,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T09:38:45,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:45,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:45,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:45,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:45,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:45,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:45,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742431_1607 (size=12151) 2024-11-07T09:38:45,328 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/04bd274447bb490c937f47c810c4d086 2024-11-07T09:38:45,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/b3629c9fc1594fc8a404e28e04f41d8d is 50, key is test_row_0/C:col10/1730972323949/Put/seqid=0 2024-11-07T09:38:45,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T09:38:45,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:45,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:45,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:45,388 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:45,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:45,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:45,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-07T09:38:45,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:45,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1730972385503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:45,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:45,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46604 deadline: 1730972385504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:45,519 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:45,519 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:45,519 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:45,520 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:45,520 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:45,520 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:45,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742432_1608 (size=12151) 2024-11-07T09:38:45,541 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/b3629c9fc1594fc8a404e28e04f41d8d 2024-11-07T09:38:45,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T09:38:45,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:45,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:45,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:45,549 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:45,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:45,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:45,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/9f11bbb34a9e4565a95ea7328e93da11 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/9f11bbb34a9e4565a95ea7328e93da11 2024-11-07T09:38:45,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/9f11bbb34a9e4565a95ea7328e93da11, entries=150, sequenceid=172, filesize=30.4 K 2024-11-07T09:38:45,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/04bd274447bb490c937f47c810c4d086 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/04bd274447bb490c937f47c810c4d086 2024-11-07T09:38:45,585 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/04bd274447bb490c937f47c810c4d086, entries=150, sequenceid=172, filesize=11.9 K 2024-11-07T09:38:45,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/b3629c9fc1594fc8a404e28e04f41d8d as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/b3629c9fc1594fc8a404e28e04f41d8d 2024-11-07T09:38:45,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/b3629c9fc1594fc8a404e28e04f41d8d, entries=150, sequenceid=172, filesize=11.9 K 2024-11-07T09:38:45,660 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for e4cc6107007e595af5a0df7b5bc3711a in 1710ms, sequenceid=172, compaction requested=false 2024-11-07T09:38:45,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e4cc6107007e595af5a0df7b5bc3711a: 2024-11-07T09:38:45,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:45,677 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:38:45,678 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e4cc6107007e595af5a0df7b5bc3711a 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-07T09:38:45,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T09:38:45,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:45,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:45,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:45,704 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:45,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:45,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:45,743 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411076f0628fe31ef4035b847c5ed9fffc8e4_e4cc6107007e595af5a0df7b5bc3711a is 50, key is test_row_0/A:col10/1730972324199/Put/seqid=0 2024-11-07T09:38:45,875 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T09:38:45,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:45,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:45,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:45,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:45,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:45,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:45,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:45,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46592 deadline: 1730972385835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:45,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:45,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:45,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:45,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:45,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46550 deadline: 1730972385835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:45,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:45,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:45,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:45,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742433_1609 (size=14794) 2024-11-07T09:38:46,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:46,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46592 deadline: 1730972385983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:46,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:46,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:46,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:46,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:46,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46550 deadline: 1730972385983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:46,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:46,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:46,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:46,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T09:38:46,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:46,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:46,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:46,086 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:46,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:46,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:46,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T09:38:46,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:46,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:46,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:46,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:46,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:46,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:46,313 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:46,325 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411076f0628fe31ef4035b847c5ed9fffc8e4_e4cc6107007e595af5a0df7b5bc3711a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411076f0628fe31ef4035b847c5ed9fffc8e4_e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:46,329 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/6c7e40889c09437a8e4068195fa5910e, store: [table=TestAcidGuarantees family=A region=e4cc6107007e595af5a0df7b5bc3711a] 2024-11-07T09:38:46,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/6c7e40889c09437a8e4068195fa5910e is 175, key is test_row_0/A:col10/1730972324199/Put/seqid=0 2024-11-07T09:38:46,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T09:38:46,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:46,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:46,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:46,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:46,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:46,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:46,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:46,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46550 deadline: 1730972386391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:46,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:46,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:46,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:46,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:46,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46592 deadline: 1730972386391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:46,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:46,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:46,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:46,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742434_1610 (size=39749) 2024-11-07T09:38:46,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-07T09:38:46,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:46,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46604 deadline: 1730972386522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:46,537 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 9f1eb259a29b,42895,1730972153970 for 'test_row_0' in e4cc6107007e595af5a0df7b5bc3711a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2337 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T09:38:46,538 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:46,538 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:46,538 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:46,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:46,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1730972386535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:46,557 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 9f1eb259a29b,42895,1730972153970 for 'test_row_0' in e4cc6107007e595af5a0df7b5bc3711a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2362 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T09:38:46,558 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:46,558 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:46,558 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:46,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T09:38:46,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:46,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:46,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:46,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:46,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:46,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:46,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:46,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46550 deadline: 1730972386733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:46,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:46,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T09:38:46,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:46,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:46,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:46,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:46,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:46,753 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:46,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:46,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:46,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:46,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46592 deadline: 1730972386763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:46,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:46,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:46,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:46,870 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=202, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/6c7e40889c09437a8e4068195fa5910e 2024-11-07T09:38:46,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/dcb0b445b9ac4d51a4903cc1e290ae32 is 50, key is test_row_0/B:col10/1730972324199/Put/seqid=0 2024-11-07T09:38:46,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T09:38:46,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:46,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:46,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:46,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:46,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:46,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:47,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T09:38:47,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:47,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:47,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:47,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:47,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:47,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:47,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742435_1611 (size=12151) 2024-11-07T09:38:47,258 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/dcb0b445b9ac4d51a4903cc1e290ae32 2024-11-07T09:38:47,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:47,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46550 deadline: 1730972387263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:47,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:47,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:47,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:47,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T09:38:47,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:47,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:47,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:47,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:47,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:47,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:47,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:47,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46592 deadline: 1730972387303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:47,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:47,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:47,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:47,320 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/33ffc27a03b04fbc9f516bb4c460cc9d is 50, key is test_row_0/C:col10/1730972324199/Put/seqid=0 2024-11-07T09:38:47,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T09:38:47,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:47,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:47,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:47,478 ERROR [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:47,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:47,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T09:38:47,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742436_1612 (size=12151) 2024-11-07T09:38:47,485 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/33ffc27a03b04fbc9f516bb4c460cc9d 2024-11-07T09:38:47,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/6c7e40889c09437a8e4068195fa5910e as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/6c7e40889c09437a8e4068195fa5910e 2024-11-07T09:38:47,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/6c7e40889c09437a8e4068195fa5910e, entries=200, sequenceid=202, filesize=38.8 K 2024-11-07T09:38:47,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/dcb0b445b9ac4d51a4903cc1e290ae32 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/dcb0b445b9ac4d51a4903cc1e290ae32 2024-11-07T09:38:47,498 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/dcb0b445b9ac4d51a4903cc1e290ae32, entries=150, sequenceid=202, filesize=11.9 K 2024-11-07T09:38:47,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/33ffc27a03b04fbc9f516bb4c460cc9d as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/33ffc27a03b04fbc9f516bb4c460cc9d 2024-11-07T09:38:47,515 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/33ffc27a03b04fbc9f516bb4c460cc9d, entries=150, sequenceid=202, filesize=11.9 K 2024-11-07T09:38:47,516 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for e4cc6107007e595af5a0df7b5bc3711a in 1839ms, sequenceid=202, compaction requested=true 2024-11-07T09:38:47,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e4cc6107007e595af5a0df7b5bc3711a: 2024-11-07T09:38:47,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4cc6107007e595af5a0df7b5bc3711a:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T09:38:47,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:47,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4cc6107007e595af5a0df7b5bc3711a:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T09:38:47,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:38:47,516 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:38:47,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4cc6107007e595af5a0df7b5bc3711a:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:38:47,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T09:38:47,517 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:38:47,517 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:38:47,518 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): e4cc6107007e595af5a0df7b5bc3711a/A is initiating minor compaction (all files) 2024-11-07T09:38:47,518 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4cc6107007e595af5a0df7b5bc3711a/A in TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:47,518 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/6d14692d71254e929fc7bdee946f952a, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/9f11bbb34a9e4565a95ea7328e93da11, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/6c7e40889c09437a8e4068195fa5910e] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp, totalSize=99.9 K 2024-11-07T09:38:47,518 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=65.38 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:47,518 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. files: [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/6d14692d71254e929fc7bdee946f952a, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/9f11bbb34a9e4565a95ea7328e93da11, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/6c7e40889c09437a8e4068195fa5910e] 2024-11-07T09:38:47,518 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:38:47,518 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): e4cc6107007e595af5a0df7b5bc3711a/B is initiating minor compaction (all files) 2024-11-07T09:38:47,518 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4cc6107007e595af5a0df7b5bc3711a/B in TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:47,518 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/ef0910e7bd9840838d641dbd982c1111, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/04bd274447bb490c937f47c810c4d086, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/dcb0b445b9ac4d51a4903cc1e290ae32] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp, totalSize=35.9 K 2024-11-07T09:38:47,519 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d14692d71254e929fc7bdee946f952a, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1730972322080 2024-11-07T09:38:47,519 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting ef0910e7bd9840838d641dbd982c1111, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1730972322080 2024-11-07T09:38:47,519 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f11bbb34a9e4565a95ea7328e93da11, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1730972322468 2024-11-07T09:38:47,520 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 04bd274447bb490c937f47c810c4d086, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1730972322468 2024-11-07T09:38:47,520 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c7e40889c09437a8e4068195fa5910e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1730972324192 2024-11-07T09:38:47,521 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting dcb0b445b9ac4d51a4903cc1e290ae32, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1730972324192 2024-11-07T09:38:47,549 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e4cc6107007e595af5a0df7b5bc3711a] 2024-11-07T09:38:47,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,602 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107719d0a688a2c4b5f86ab3cfc3dd561c8_e4cc6107007e595af5a0df7b5bc3711a store=[table=TestAcidGuarantees family=A region=e4cc6107007e595af5a0df7b5bc3711a] 2024-11-07T09:38:47,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,604 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4cc6107007e595af5a0df7b5bc3711a#B#compaction#525 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 65.38 MB/second 2024-11-07T09:38:47,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,604 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/5e043fa519d441549e1d8741cc3e49a2 is 50, key is test_row_0/B:col10/1730972324199/Put/seqid=0 2024-11-07T09:38:47,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,606 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107719d0a688a2c4b5f86ab3cfc3dd561c8_e4cc6107007e595af5a0df7b5bc3711a, store=[table=TestAcidGuarantees family=A region=e4cc6107007e595af5a0df7b5bc3711a] 2024-11-07T09:38:47,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,606 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107719d0a688a2c4b5f86ab3cfc3dd561c8_e4cc6107007e595af5a0df7b5bc3711a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e4cc6107007e595af5a0df7b5bc3711a] 2024-11-07T09:38:47,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42895 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T09:38:47,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:47,634 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing e4cc6107007e595af5a0df7b5bc3711a 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-07T09:38:47,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742437_1613 (size=12595) 2024-11-07T09:38:47,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,864 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/5e043fa519d441549e1d8741cc3e49a2 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/5e043fa519d441549e1d8741cc3e49a2 2024-11-07T09:38:47,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,869 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e4cc6107007e595af5a0df7b5bc3711a/B of e4cc6107007e595af5a0df7b5bc3711a into 5e043fa519d441549e1d8741cc3e49a2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:38:47,869 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4cc6107007e595af5a0df7b5bc3711a: 2024-11-07T09:38:47,869 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., storeName=e4cc6107007e595af5a0df7b5bc3711a/B, priority=13, startTime=1730972327516; duration=0sec 2024-11-07T09:38:47,869 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:38:47,869 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4cc6107007e595af5a0df7b5bc3711a:B 2024-11-07T09:38:47,869 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:38:47,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107b07575debecd4df1a906f4c1604a0110_e4cc6107007e595af5a0df7b5bc3711a is 50, key is test_row_1/A:col10/1730972325809/Put/seqid=0 2024-11-07T09:38:47,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,870 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:38:47,870 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): e4cc6107007e595af5a0df7b5bc3711a/C is initiating minor compaction (all files) 2024-11-07T09:38:47,870 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4cc6107007e595af5a0df7b5bc3711a/C in TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:47,871 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/92ece59a5a3745c79b39542e651799ed, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/b3629c9fc1594fc8a404e28e04f41d8d, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/33ffc27a03b04fbc9f516bb4c460cc9d] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp, totalSize=35.9 K 2024-11-07T09:38:47,871 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 92ece59a5a3745c79b39542e651799ed, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1730972322080 2024-11-07T09:38:47,872 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting b3629c9fc1594fc8a404e28e04f41d8d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1730972322468 2024-11-07T09:38:47,872 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 33ffc27a03b04fbc9f516bb4c460cc9d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1730972324192 2024-11-07T09:38:47,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742438_1614 (size=4469) 2024-11-07T09:38:47,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,926 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4cc6107007e595af5a0df7b5bc3711a#A#compaction#524 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 65.38 MB/second 2024-11-07T09:38:47,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,927 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/577623b0d50740d4bc469948bd92e915 is 175, key is test_row_0/A:col10/1730972324199/Put/seqid=0 2024-11-07T09:38:47,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,966 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4cc6107007e595af5a0df7b5bc3711a#C#compaction#527 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 65.38 MB/second 2024-11-07T09:38:47,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,966 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/1c8a8ff222304f5eb5183a796551f815 is 50, key is test_row_0/C:col10/1730972324199/Put/seqid=0 2024-11-07T09:38:47,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:47,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742439_1615 (size=9814) 2024-11-07T09:38:48,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,076 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107b07575debecd4df1a906f4c1604a0110_e4cc6107007e595af5a0df7b5bc3711a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107b07575debecd4df1a906f4c1604a0110_e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:48,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/09098679acd0404ca045d397ac0036f2, store: [table=TestAcidGuarantees family=A region=e4cc6107007e595af5a0df7b5bc3711a] 2024-11-07T09:38:48,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/09098679acd0404ca045d397ac0036f2 is 175, key is test_row_1/A:col10/1730972325809/Put/seqid=0 2024-11-07T09:38:48,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742441_1617 (size=12595) 2024-11-07T09:38:48,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742442_1618 (size=22461) 2024-11-07T09:38:48,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742440_1616 (size=31549) 2024-11-07T09:38:48,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:48,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. as already flushing 2024-11-07T09:38:48,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-07T09:38:48,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,596 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=211, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/09098679acd0404ca045d397ac0036f2 2024-11-07T09:38:48,602 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/1c8a8ff222304f5eb5183a796551f815 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/1c8a8ff222304f5eb5183a796551f815 2024-11-07T09:38:48,604 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/577623b0d50740d4bc469948bd92e915 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/577623b0d50740d4bc469948bd92e915 2024-11-07T09:38:48,608 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e4cc6107007e595af5a0df7b5bc3711a/C of e4cc6107007e595af5a0df7b5bc3711a into 1c8a8ff222304f5eb5183a796551f815(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:38:48,608 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4cc6107007e595af5a0df7b5bc3711a: 2024-11-07T09:38:48,608 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., storeName=e4cc6107007e595af5a0df7b5bc3711a/C, priority=13, startTime=1730972327516; duration=0sec 2024-11-07T09:38:48,608 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:48,608 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4cc6107007e595af5a0df7b5bc3711a:C 2024-11-07T09:38:48,609 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e4cc6107007e595af5a0df7b5bc3711a/A of e4cc6107007e595af5a0df7b5bc3711a into 577623b0d50740d4bc469948bd92e915(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 1sec to execute. 2024-11-07T09:38:48,609 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4cc6107007e595af5a0df7b5bc3711a: 2024-11-07T09:38:48,609 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., storeName=e4cc6107007e595af5a0df7b5bc3711a/A, priority=13, startTime=1730972327516; duration=1sec 2024-11-07T09:38:48,609 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:48,609 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4cc6107007e595af5a0df7b5bc3711a:A 2024-11-07T09:38:48,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/2155445caa6b44ab804b6d35f57e1c4f is 50, key is test_row_1/B:col10/1730972325809/Put/seqid=0 2024-11-07T09:38:48,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742443_1619 (size=9757) 2024-11-07T09:38:48,685 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/2155445caa6b44ab804b6d35f57e1c4f 2024-11-07T09:38:48,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/2a64c9dea9564cb0acd3e5af3c1ba7ba is 50, key is test_row_1/C:col10/1730972325809/Put/seqid=0 2024-11-07T09:38:48,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:48,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46604 deadline: 1730972388736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:48,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:48,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:48,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:48,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:48,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1730972388744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:48,774 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:48,774 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:48,774 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:48,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:48,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46550 deadline: 1730972388750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:48,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:48,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:48,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:48,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:48,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742444_1620 (size=9757) 2024-11-07T09:38:48,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46592 deadline: 1730972388751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:48,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:48,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:48,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:48,785 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/2a64c9dea9564cb0acd3e5af3c1ba7ba 2024-11-07T09:38:48,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/09098679acd0404ca045d397ac0036f2 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/09098679acd0404ca045d397ac0036f2 2024-11-07T09:38:48,795 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/09098679acd0404ca045d397ac0036f2, entries=100, sequenceid=211, filesize=21.9 K 2024-11-07T09:38:48,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/2155445caa6b44ab804b6d35f57e1c4f as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/2155445caa6b44ab804b6d35f57e1c4f 2024-11-07T09:38:48,801 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/2155445caa6b44ab804b6d35f57e1c4f, entries=100, sequenceid=211, filesize=9.5 K 2024-11-07T09:38:48,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/2a64c9dea9564cb0acd3e5af3c1ba7ba as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/2a64c9dea9564cb0acd3e5af3c1ba7ba 2024-11-07T09:38:48,805 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/2a64c9dea9564cb0acd3e5af3c1ba7ba, entries=100, sequenceid=211, filesize=9.5 K 2024-11-07T09:38:48,811 INFO [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for e4cc6107007e595af5a0df7b5bc3711a in 1177ms, sequenceid=211, compaction requested=false 2024-11-07T09:38:48,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for e4cc6107007e595af5a0df7b5bc3711a: 2024-11-07T09:38:48,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:48,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9f1eb259a29b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-07T09:38:48,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-07T09:38:48,814 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-07T09:38:48,814 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.4440 sec 2024-11-07T09:38:48,816 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 4.4490 sec 2024-11-07T09:38:48,885 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:38:48,885 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e4cc6107007e595af5a0df7b5bc3711a 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-07T09:38:48,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:48,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:48,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46592 deadline: 1730972388895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:48,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:48,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:48,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:48,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:48,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46604 deadline: 1730972388905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:48,912 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:48,912 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:48,912 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:48,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:48,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1730972388906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:48,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:48,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46550 deadline: 1730972388910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:48,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:48,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:48,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:48,915 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:48,915 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:48,915 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:48,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107aa3c85f41c4c45df93b60757e862cb59_e4cc6107007e595af5a0df7b5bc3711a is 50, key is test_row_0/A:col10/1730972328728/Put/seqid=0 2024-11-07T09:38:48,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742445_1621 (size=14794) 2024-11-07T09:38:48,956 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:48,960 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107aa3c85f41c4c45df93b60757e862cb59_e4cc6107007e595af5a0df7b5bc3711a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107aa3c85f41c4c45df93b60757e862cb59_e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:48,961 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/5d149ba83d034089ab21393fba0af980, store: [table=TestAcidGuarantees family=A region=e4cc6107007e595af5a0df7b5bc3711a] 2024-11-07T09:38:48,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/5d149ba83d034089ab21393fba0af980 is 175, key is test_row_0/A:col10/1730972328728/Put/seqid=0 2024-11-07T09:38:49,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742446_1622 (size=39749) 2024-11-07T09:38:49,014 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=242, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/5d149ba83d034089ab21393fba0af980 2024-11-07T09:38:49,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:49,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46592 deadline: 1730972389013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:49,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:49,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:49,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:49,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:49,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46604 deadline: 1730972389023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:49,027 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:49,027 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:49,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:49,027 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:49,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46550 deadline: 1730972389023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:49,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:49,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:49,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:49,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:49,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1730972389023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:49,029 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:49,029 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:49,029 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:49,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/ac8a88df983f43d6943705cee439911e is 50, key is test_row_0/B:col10/1730972328728/Put/seqid=0 2024-11-07T09:38:49,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742447_1623 (size=12151) 2024-11-07T09:38:49,057 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/ac8a88df983f43d6943705cee439911e 2024-11-07T09:38:49,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/bfec82264ccb40edb292cc759371a437 is 50, key is test_row_0/C:col10/1730972328728/Put/seqid=0 2024-11-07T09:38:49,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742448_1624 (size=12151) 2024-11-07T09:38:49,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/bfec82264ccb40edb292cc759371a437 2024-11-07T09:38:49,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/5d149ba83d034089ab21393fba0af980 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/5d149ba83d034089ab21393fba0af980 2024-11-07T09:38:49,105 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/5d149ba83d034089ab21393fba0af980, entries=200, sequenceid=242, filesize=38.8 K 2024-11-07T09:38:49,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/ac8a88df983f43d6943705cee439911e as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/ac8a88df983f43d6943705cee439911e 2024-11-07T09:38:49,111 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/ac8a88df983f43d6943705cee439911e, entries=150, sequenceid=242, filesize=11.9 K 2024-11-07T09:38:49,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/bfec82264ccb40edb292cc759371a437 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/bfec82264ccb40edb292cc759371a437 2024-11-07T09:38:49,117 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/bfec82264ccb40edb292cc759371a437, entries=150, sequenceid=242, filesize=11.9 K 2024-11-07T09:38:49,119 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=40.25 KB/41220 for e4cc6107007e595af5a0df7b5bc3711a in 234ms, sequenceid=242, compaction requested=true 2024-11-07T09:38:49,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e4cc6107007e595af5a0df7b5bc3711a: 2024-11-07T09:38:49,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4cc6107007e595af5a0df7b5bc3711a:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T09:38:49,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:49,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4cc6107007e595af5a0df7b5bc3711a:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T09:38:49,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:38:49,119 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:38:49,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4cc6107007e595af5a0df7b5bc3711a:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:38:49,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T09:38:49,119 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:38:49,120 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:38:49,120 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): e4cc6107007e595af5a0df7b5bc3711a/B is initiating minor compaction (all files) 2024-11-07T09:38:49,120 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4cc6107007e595af5a0df7b5bc3711a/B in TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:49,120 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/5e043fa519d441549e1d8741cc3e49a2, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/2155445caa6b44ab804b6d35f57e1c4f, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/ac8a88df983f43d6943705cee439911e] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp, totalSize=33.7 K 2024-11-07T09:38:49,121 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93759 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:38:49,121 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e043fa519d441549e1d8741cc3e49a2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1730972324192 2024-11-07T09:38:49,121 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): e4cc6107007e595af5a0df7b5bc3711a/A is initiating minor compaction (all files) 2024-11-07T09:38:49,121 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4cc6107007e595af5a0df7b5bc3711a/A in TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:49,121 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/577623b0d50740d4bc469948bd92e915, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/09098679acd0404ca045d397ac0036f2, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/5d149ba83d034089ab21393fba0af980] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp, totalSize=91.6 K 2024-11-07T09:38:49,121 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=65.38 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:49,121 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. files: [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/577623b0d50740d4bc469948bd92e915, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/09098679acd0404ca045d397ac0036f2, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/5d149ba83d034089ab21393fba0af980] 2024-11-07T09:38:49,121 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 2155445caa6b44ab804b6d35f57e1c4f, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1730972325772 2024-11-07T09:38:49,122 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 577623b0d50740d4bc469948bd92e915, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1730972324192 2024-11-07T09:38:49,122 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting ac8a88df983f43d6943705cee439911e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1730972328672 2024-11-07T09:38:49,122 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09098679acd0404ca045d397ac0036f2, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1730972325772 2024-11-07T09:38:49,122 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d149ba83d034089ab21393fba0af980, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1730972328672 2024-11-07T09:38:49,134 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4cc6107007e595af5a0df7b5bc3711a#B#compaction#533 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 65.38 MB/second 2024-11-07T09:38:49,134 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/b1f85386ea92425ebbf974ad9ba07962 is 50, key is test_row_0/B:col10/1730972328728/Put/seqid=0 2024-11-07T09:38:49,135 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e4cc6107007e595af5a0df7b5bc3711a] 2024-11-07T09:38:49,153 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107838ff7de49554a9fbb5a8ffe245f9ade_e4cc6107007e595af5a0df7b5bc3711a store=[table=TestAcidGuarantees family=A region=e4cc6107007e595af5a0df7b5bc3711a] 2024-11-07T09:38:49,155 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107838ff7de49554a9fbb5a8ffe245f9ade_e4cc6107007e595af5a0df7b5bc3711a, store=[table=TestAcidGuarantees family=A region=e4cc6107007e595af5a0df7b5bc3711a] 2024-11-07T09:38:49,155 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107838ff7de49554a9fbb5a8ffe245f9ade_e4cc6107007e595af5a0df7b5bc3711a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e4cc6107007e595af5a0df7b5bc3711a] 2024-11-07T09:38:49,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742449_1625 (size=12697) 2024-11-07T09:38:49,164 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/b1f85386ea92425ebbf974ad9ba07962 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/b1f85386ea92425ebbf974ad9ba07962 2024-11-07T09:38:49,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742450_1626 (size=4469) 2024-11-07T09:38:49,169 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e4cc6107007e595af5a0df7b5bc3711a/B of e4cc6107007e595af5a0df7b5bc3711a into b1f85386ea92425ebbf974ad9ba07962(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:38:49,169 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4cc6107007e595af5a0df7b5bc3711a: 2024-11-07T09:38:49,169 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., storeName=e4cc6107007e595af5a0df7b5bc3711a/B, priority=13, startTime=1730972329119; duration=0sec 2024-11-07T09:38:49,169 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:38:49,169 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4cc6107007e595af5a0df7b5bc3711a:B 2024-11-07T09:38:49,169 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:38:49,169 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4cc6107007e595af5a0df7b5bc3711a#A#compaction#534 average throughput is 0.72 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 65.38 MB/second 2024-11-07T09:38:49,170 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/6e45489eb78c4810bf6d210235f413ba is 175, key is test_row_0/A:col10/1730972328728/Put/seqid=0 2024-11-07T09:38:49,171 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:38:49,171 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): e4cc6107007e595af5a0df7b5bc3711a/C is initiating minor compaction (all files) 2024-11-07T09:38:49,171 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4cc6107007e595af5a0df7b5bc3711a/C in TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:49,171 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/1c8a8ff222304f5eb5183a796551f815, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/2a64c9dea9564cb0acd3e5af3c1ba7ba, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/bfec82264ccb40edb292cc759371a437] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp, totalSize=33.7 K 2024-11-07T09:38:49,172 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c8a8ff222304f5eb5183a796551f815, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1730972324192 2024-11-07T09:38:49,172 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a64c9dea9564cb0acd3e5af3c1ba7ba, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1730972325772 2024-11-07T09:38:49,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742451_1627 (size=31651) 2024-11-07T09:38:49,173 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting bfec82264ccb40edb292cc759371a437, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1730972328672 2024-11-07T09:38:49,180 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/6e45489eb78c4810bf6d210235f413ba as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/6e45489eb78c4810bf6d210235f413ba 2024-11-07T09:38:49,184 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4cc6107007e595af5a0df7b5bc3711a#C#compaction#535 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 65.38 MB/second 2024-11-07T09:38:49,185 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/46eedc2f70c64a19833be0ca75c1a921 is 50, key is test_row_0/C:col10/1730972328728/Put/seqid=0 2024-11-07T09:38:49,188 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e4cc6107007e595af5a0df7b5bc3711a/A of e4cc6107007e595af5a0df7b5bc3711a into 6e45489eb78c4810bf6d210235f413ba(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:38:49,188 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4cc6107007e595af5a0df7b5bc3711a: 2024-11-07T09:38:49,188 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., storeName=e4cc6107007e595af5a0df7b5bc3711a/A, priority=13, startTime=1730972329119; duration=0sec 2024-11-07T09:38:49,188 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:49,189 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4cc6107007e595af5a0df7b5bc3711a:A 2024-11-07T09:38:49,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742452_1628 (size=12697) 2024-11-07T09:38:49,209 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/46eedc2f70c64a19833be0ca75c1a921 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/46eedc2f70c64a19833be0ca75c1a921 2024-11-07T09:38:49,215 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e4cc6107007e595af5a0df7b5bc3711a/C of e4cc6107007e595af5a0df7b5bc3711a into 46eedc2f70c64a19833be0ca75c1a921(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:38:49,215 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4cc6107007e595af5a0df7b5bc3711a: 2024-11-07T09:38:49,215 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., storeName=e4cc6107007e595af5a0df7b5bc3711a/C, priority=13, startTime=1730972329119; duration=0sec 2024-11-07T09:38:49,216 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:49,216 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4cc6107007e595af5a0df7b5bc3711a:C 2024-11-07T09:38:49,233 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:38:49,233 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e4cc6107007e595af5a0df7b5bc3711a 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T09:38:49,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:49,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107ffb48b84a0da4370a463a72e3de00262_e4cc6107007e595af5a0df7b5bc3711a is 50, key is test_row_0/A:col10/1730972329230/Put/seqid=0 2024-11-07T09:38:49,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742453_1629 (size=14794) 2024-11-07T09:38:49,259 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:49,263 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107ffb48b84a0da4370a463a72e3de00262_e4cc6107007e595af5a0df7b5bc3711a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107ffb48b84a0da4370a463a72e3de00262_e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:49,263 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/43a83c3ded31437087c01d85b22d16fc, store: [table=TestAcidGuarantees family=A region=e4cc6107007e595af5a0df7b5bc3711a] 2024-11-07T09:38:49,264 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/43a83c3ded31437087c01d85b22d16fc is 175, key is test_row_0/A:col10/1730972329230/Put/seqid=0 2024-11-07T09:38:49,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:49,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46592 deadline: 1730972389266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:49,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:49,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:49,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:49,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:49,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1730972389268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:49,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:49,274 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:49,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46550 deadline: 1730972389269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:49,274 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:49,274 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:49,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:49,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46604 deadline: 1730972389269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:49,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:49,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:49,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:49,275 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:49,275 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:49,275 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:49,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742454_1630 (size=39749) 2024-11-07T09:38:49,275 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=256, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/43a83c3ded31437087c01d85b22d16fc 2024-11-07T09:38:49,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/6743acf1504444e6b6beb94b56b67e4c is 50, key is test_row_0/B:col10/1730972329230/Put/seqid=0 2024-11-07T09:38:49,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742455_1631 (size=12151) 2024-11-07T09:38:49,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:49,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46592 deadline: 1730972389373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:49,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:49,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:49,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:49,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:49,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1730972389382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:49,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:49,386 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:49,386 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:49,386 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:49,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46604 deadline: 1730972389383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:49,386 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:49,386 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:49,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:49,386 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:49,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46550 deadline: 1730972389383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:49,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:49,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:49,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:49,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:49,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46592 deadline: 1730972389582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:49,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:49,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:49,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:49,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:49,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1730972389593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:49,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:49,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:49,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46604 deadline: 1730972389593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:49,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46550 deadline: 1730972389593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:49,597 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:49,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:49,597 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:49,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:49,597 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:49,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:49,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:49,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:49,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:49,693 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/6743acf1504444e6b6beb94b56b67e4c 2024-11-07T09:38:49,700 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/16c1e08343bb487babbe80aa7777e981 is 50, key is test_row_0/C:col10/1730972329230/Put/seqid=0 2024-11-07T09:38:49,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742456_1632 (size=12151) 2024-11-07T09:38:49,705 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/16c1e08343bb487babbe80aa7777e981 2024-11-07T09:38:49,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/43a83c3ded31437087c01d85b22d16fc as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/43a83c3ded31437087c01d85b22d16fc 2024-11-07T09:38:49,713 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/43a83c3ded31437087c01d85b22d16fc, entries=200, sequenceid=256, filesize=38.8 K 2024-11-07T09:38:49,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/6743acf1504444e6b6beb94b56b67e4c as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/6743acf1504444e6b6beb94b56b67e4c 2024-11-07T09:38:49,718 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/6743acf1504444e6b6beb94b56b67e4c, entries=150, sequenceid=256, filesize=11.9 K 2024-11-07T09:38:49,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/16c1e08343bb487babbe80aa7777e981 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/16c1e08343bb487babbe80aa7777e981 2024-11-07T09:38:49,721 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/16c1e08343bb487babbe80aa7777e981, entries=150, sequenceid=256, filesize=11.9 K 2024-11-07T09:38:49,722 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for e4cc6107007e595af5a0df7b5bc3711a in 489ms, sequenceid=256, compaction requested=false 2024-11-07T09:38:49,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e4cc6107007e595af5a0df7b5bc3711a: 2024-11-07T09:38:49,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:49,896 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:38:49,896 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e4cc6107007e595af5a0df7b5bc3711a 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-07T09:38:49,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107db1619780e454839a3eaf863cdcd99b3_e4cc6107007e595af5a0df7b5bc3711a is 50, key is test_row_0/A:col10/1730972329259/Put/seqid=0 2024-11-07T09:38:49,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742457_1633 (size=14994) 2024-11-07T09:38:49,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:49,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46550 deadline: 1730972389909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:49,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:49,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:49,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:49,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:49,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1730972389915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:49,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:49,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46592 deadline: 1730972389916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:49,920 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:49,920 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:49,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:49,920 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:49,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:49,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:49,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:49,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46604 deadline: 1730972389916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:49,921 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:49,921 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:49,921 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:50,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:50,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:50,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1730972390023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:50,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46550 deadline: 1730972390023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:50,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:50,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:50,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46592 deadline: 1730972390023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:50,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:50,027 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:50,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46604 deadline: 1730972390023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:50,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:50,027 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:50,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:50,027 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:50,027 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:50,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:50,027 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:50,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:50,027 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:50,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:50,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:50,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:50,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46550 deadline: 1730972390233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:50,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1730972390233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:50,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:50,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:50,236 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:50,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46604 deadline: 1730972390233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:50,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:50,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:50,236 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:50,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:50,237 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:50,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46592 deadline: 1730972390233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:50,237 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:50,237 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:50,237 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:50,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:50,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:50,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:50,307 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:50,310 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107db1619780e454839a3eaf863cdcd99b3_e4cc6107007e595af5a0df7b5bc3711a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107db1619780e454839a3eaf863cdcd99b3_e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:50,311 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/eb2c4d17f6db433888714658185f78ca, store: [table=TestAcidGuarantees family=A region=e4cc6107007e595af5a0df7b5bc3711a] 2024-11-07T09:38:50,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/eb2c4d17f6db433888714658185f78ca is 175, key is test_row_0/A:col10/1730972329259/Put/seqid=0 2024-11-07T09:38:50,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742458_1634 (size=39949) 2024-11-07T09:38:50,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:50,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46550 deadline: 1730972390542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:50,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:50,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46604 deadline: 1730972390543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:50,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:50,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:50,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:50,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:50,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1730972390543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:50,543 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:50,543 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:50,543 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:50,544 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:50,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:50,544 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:50,544 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:50,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46592 deadline: 1730972390543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:50,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:50,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:50,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:50,716 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=282, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/eb2c4d17f6db433888714658185f78ca 2024-11-07T09:38:50,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/edccb37e73894a9cb78a30f9df182937 is 50, key is test_row_0/B:col10/1730972329259/Put/seqid=0 2024-11-07T09:38:50,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742459_1635 (size=12301) 2024-11-07T09:38:51,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:51,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46550 deadline: 1730972391053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:51,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:51,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46604 deadline: 1730972391053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:51,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:51,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:51,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:51,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:51,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46630 deadline: 1730972391053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:51,057 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:51,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T09:38:51,057 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:51,057 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:51,057 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:51,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46592 deadline: 1730972391053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:51,057 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:51,057 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:51,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 , the old value is region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:51,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T09:38:51,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., hostname=9f1eb259a29b,42895,1730972153970, seqNum=5 because the exception is null or not the one we care about 2024-11-07T09:38:51,126 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/edccb37e73894a9cb78a30f9df182937 2024-11-07T09:38:51,133 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/809c3cc3d62d4cc7ac25b7fc27a2dcc0 is 50, key is test_row_0/C:col10/1730972329259/Put/seqid=0 2024-11-07T09:38:51,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742460_1636 (size=12301) 2024-11-07T09:38:51,538 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/809c3cc3d62d4cc7ac25b7fc27a2dcc0 2024-11-07T09:38:51,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/eb2c4d17f6db433888714658185f78ca as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/eb2c4d17f6db433888714658185f78ca 2024-11-07T09:38:51,544 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/eb2c4d17f6db433888714658185f78ca, entries=200, sequenceid=282, filesize=39.0 K 2024-11-07T09:38:51,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/edccb37e73894a9cb78a30f9df182937 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/edccb37e73894a9cb78a30f9df182937 2024-11-07T09:38:51,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,547 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/edccb37e73894a9cb78a30f9df182937, entries=150, sequenceid=282, filesize=12.0 K 2024-11-07T09:38:51,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/809c3cc3d62d4cc7ac25b7fc27a2dcc0 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/809c3cc3d62d4cc7ac25b7fc27a2dcc0 2024-11-07T09:38:51,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/809c3cc3d62d4cc7ac25b7fc27a2dcc0, entries=150, sequenceid=282, filesize=12.0 K 2024-11-07T09:38:51,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,552 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for e4cc6107007e595af5a0df7b5bc3711a in 1656ms, sequenceid=282, compaction requested=true 2024-11-07T09:38:51,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e4cc6107007e595af5a0df7b5bc3711a: 2024-11-07T09:38:51,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4cc6107007e595af5a0df7b5bc3711a:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T09:38:51,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:51,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4cc6107007e595af5a0df7b5bc3711a:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T09:38:51,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:51,552 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:38:51,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e4cc6107007e595af5a0df7b5bc3711a:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T09:38:51,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:38:51,552 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:38:51,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,553 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:38:51,553 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111349 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:38:51,553 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): e4cc6107007e595af5a0df7b5bc3711a/B is initiating minor compaction (all files) 2024-11-07T09:38:51,553 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1541): e4cc6107007e595af5a0df7b5bc3711a/A is initiating minor compaction (all files) 2024-11-07T09:38:51,553 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4cc6107007e595af5a0df7b5bc3711a/B in TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:51,553 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/b1f85386ea92425ebbf974ad9ba07962, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/6743acf1504444e6b6beb94b56b67e4c, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/edccb37e73894a9cb78a30f9df182937] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp, totalSize=36.3 K 2024-11-07T09:38:51,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,553 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4cc6107007e595af5a0df7b5bc3711a/A in TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:51,553 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/6e45489eb78c4810bf6d210235f413ba, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/43a83c3ded31437087c01d85b22d16fc, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/eb2c4d17f6db433888714658185f78ca] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp, totalSize=108.7 K 2024-11-07T09:38:51,553 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=65.38 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:51,553 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting b1f85386ea92425ebbf974ad9ba07962, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1730972328672 2024-11-07T09:38:51,553 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. files: [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/6e45489eb78c4810bf6d210235f413ba, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/43a83c3ded31437087c01d85b22d16fc, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/eb2c4d17f6db433888714658185f78ca] 2024-11-07T09:38:51,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,554 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e45489eb78c4810bf6d210235f413ba, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1730972328672 2024-11-07T09:38:51,554 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 6743acf1504444e6b6beb94b56b67e4c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1730972328889 2024-11-07T09:38:51,554 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting edccb37e73894a9cb78a30f9df182937, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1730972329259 2024-11-07T09:38:51,554 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43a83c3ded31437087c01d85b22d16fc, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1730972328889 2024-11-07T09:38:51,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,554 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb2c4d17f6db433888714658185f78ca, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1730972329259 2024-11-07T09:38:51,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,560 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e4cc6107007e595af5a0df7b5bc3711a] 2024-11-07T09:38:51,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,561 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4cc6107007e595af5a0df7b5bc3711a#B#compaction#542 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 65.38 MB/second 2024-11-07T09:38:51,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,561 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107dd73abf1ddac4a6d8df9f9e3c5e4bf76_e4cc6107007e595af5a0df7b5bc3711a store=[table=TestAcidGuarantees family=A region=e4cc6107007e595af5a0df7b5bc3711a] 2024-11-07T09:38:51,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,561 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/1196bd662b8b4445ac79da797e454a59 is 50, key is test_row_0/B:col10/1730972329259/Put/seqid=0 2024-11-07T09:38:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,563 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107dd73abf1ddac4a6d8df9f9e3c5e4bf76_e4cc6107007e595af5a0df7b5bc3711a, store=[table=TestAcidGuarantees family=A region=e4cc6107007e595af5a0df7b5bc3711a] 2024-11-07T09:38:51,563 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107dd73abf1ddac4a6d8df9f9e3c5e4bf76_e4cc6107007e595af5a0df7b5bc3711a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e4cc6107007e595af5a0df7b5bc3711a] 2024-11-07T09:38:51,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742461_1637 (size=12949) 2024-11-07T09:38:51,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742462_1638 (size=4469) 2024-11-07T09:38:51,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,970 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4cc6107007e595af5a0df7b5bc3711a#A#compaction#543 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 65.38 MB/second 2024-11-07T09:38:51,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,971 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/f1848c6a168c4f4fbc6efe33989df4f4 is 175, key is test_row_0/A:col10/1730972329259/Put/seqid=0 2024-11-07T09:38:51,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,974 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/1196bd662b8b4445ac79da797e454a59 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/1196bd662b8b4445ac79da797e454a59 2024-11-07T09:38:51,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742463_1639 (size=31903) 2024-11-07T09:38:51,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,978 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e4cc6107007e595af5a0df7b5bc3711a/B of e4cc6107007e595af5a0df7b5bc3711a into 1196bd662b8b4445ac79da797e454a59(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:38:51,978 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4cc6107007e595af5a0df7b5bc3711a: 2024-11-07T09:38:51,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,978 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., storeName=e4cc6107007e595af5a0df7b5bc3711a/B, priority=13, startTime=1730972331552; duration=0sec 2024-11-07T09:38:51,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,978 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T09:38:51,978 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4cc6107007e595af5a0df7b5bc3711a:B 2024-11-07T09:38:51,978 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T09:38:51,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,979 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T09:38:51,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,979 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1541): e4cc6107007e595af5a0df7b5bc3711a/C is initiating minor compaction (all files) 2024-11-07T09:38:51,979 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e4cc6107007e595af5a0df7b5bc3711a/C in TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:51,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,979 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/46eedc2f70c64a19833be0ca75c1a921, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/16c1e08343bb487babbe80aa7777e981, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/809c3cc3d62d4cc7ac25b7fc27a2dcc0] into tmpdir=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp, totalSize=36.3 K 2024-11-07T09:38:51,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,979 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 46eedc2f70c64a19833be0ca75c1a921, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1730972328672 2024-11-07T09:38:51,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,980 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 16c1e08343bb487babbe80aa7777e981, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1730972328889 2024-11-07T09:38:51,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,980 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] compactions.Compactor(225): Compacting 809c3cc3d62d4cc7ac25b7fc27a2dcc0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1730972329259 2024-11-07T09:38:51,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,986 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e4cc6107007e595af5a0df7b5bc3711a#C#compaction#544 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 65.38 MB/second 2024-11-07T09:38:51,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,986 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/efbbe8912c944c6fb82c8005e8bfc0bb is 50, key is test_row_0/C:col10/1730972329259/Put/seqid=0 2024-11-07T09:38:51,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742464_1640 (size=12949) 2024-11-07T09:38:51,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:51,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:52,000 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/efbbe8912c944c6fb82c8005e8bfc0bb as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/efbbe8912c944c6fb82c8005e8bfc0bb 2024-11-07T09:38:52,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:52,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:52,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:52,001 INFO [Thread-2508 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2508. 2024-11-07T09:38:52,001 DEBUG [Thread-2508 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:38:52,001 DEBUG [Thread-2508 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:52,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:52,001 DEBUG [Thread-2508 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:52,001 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:38:52,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:52,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:52,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:52,002 INFO [Thread-2502 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2502. 2024-11-07T09:38:52,002 DEBUG [Thread-2502 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:38:52,002 DEBUG [Thread-2502 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:52,003 DEBUG [Thread-2502 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:52,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:52,003 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:38:52,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:52,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:52,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:52,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:52,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:52,004 INFO [Thread-2506 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2506. 2024-11-07T09:38:52,004 DEBUG [Thread-2506 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:38:52,004 DEBUG [Thread-2506 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:52,005 DEBUG [Thread-2506 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:52,005 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:38:52,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:52,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:52,005 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e4cc6107007e595af5a0df7b5bc3711a/C of e4cc6107007e595af5a0df7b5bc3711a into efbbe8912c944c6fb82c8005e8bfc0bb(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:38:52,005 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4cc6107007e595af5a0df7b5bc3711a: 2024-11-07T09:38:52,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:52,005 INFO [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., storeName=e4cc6107007e595af5a0df7b5bc3711a/C, priority=13, startTime=1730972331552; duration=0sec 2024-11-07T09:38:52,006 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:52,006 DEBUG [RS:0;9f1eb259a29b:42895-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4cc6107007e595af5a0df7b5bc3711a:C 2024-11-07T09:38:52,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:52,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:52,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:52,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:52,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:52,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:52,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:52,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:52,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:52,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:52,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:52,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:52,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:52,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:52,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:52,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:52,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:52,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:52,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:52,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:52,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:52,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:52,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:52,012 INFO [Thread-2504 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2504. 2024-11-07T09:38:52,012 DEBUG [Thread-2504 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:38:52,012 DEBUG [Thread-2504 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:52,012 DEBUG [Thread-2504 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:52,012 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:38:52,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:52,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:52,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:52,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:52,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:52,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:52,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:52,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:52,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42895 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:52,017 INFO [Thread-2510 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2510. 2024-11-07T09:38:52,017 DEBUG [Thread-2510 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:38:52,017 DEBUG [Thread-2510 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:52,017 DEBUG [Thread-2510 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:52,017 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:38:52,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42895 {}] regionserver.HRegion(8855): Flush requested on e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:52,065 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T09:38:52,065 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e4cc6107007e595af5a0df7b5bc3711a 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-07T09:38:52,065 INFO [Thread-2495 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2495. 2024-11-07T09:38:52,065 DEBUG [Thread-2495 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:38:52,065 DEBUG [Thread-2495 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:52,065 INFO [Thread-2493 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2493. 2024-11-07T09:38:52,065 DEBUG [Thread-2493 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:38:52,065 DEBUG [Thread-2493 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:52,065 DEBUG [Thread-2495 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:52,065 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:38:52,065 DEBUG [Thread-2493 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:52,065 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:38:52,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411074352fca4e3cb40cc881125575669c274_e4cc6107007e595af5a0df7b5bc3711a is 50, key is test_row_0/A:col10/1730972332063/Put/seqid=0 2024-11-07T09:38:52,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742465_1641 (size=9914) 2024-11-07T09:38:52,075 INFO [Thread-2499 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2499. 2024-11-07T09:38:52,075 INFO [Thread-2497 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2497. 2024-11-07T09:38:52,075 DEBUG [Thread-2499 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:38:52,075 DEBUG [Thread-2497 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:38:52,075 DEBUG [Thread-2499 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:52,075 DEBUG [Thread-2497 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:52,075 DEBUG [Thread-2499 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:52,075 DEBUG [Thread-2497 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:52,075 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:38:52,075 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:38:52,380 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/f1848c6a168c4f4fbc6efe33989df4f4 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/f1848c6a168c4f4fbc6efe33989df4f4 2024-11-07T09:38:52,383 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e4cc6107007e595af5a0df7b5bc3711a/A of e4cc6107007e595af5a0df7b5bc3711a into f1848c6a168c4f4fbc6efe33989df4f4(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T09:38:52,383 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e4cc6107007e595af5a0df7b5bc3711a: 2024-11-07T09:38:52,383 INFO [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., storeName=e4cc6107007e595af5a0df7b5bc3711a/A, priority=13, startTime=1730972331552; duration=0sec 2024-11-07T09:38:52,383 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T09:38:52,383 DEBUG [RS:0;9f1eb259a29b:42895-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e4cc6107007e595af5a0df7b5bc3711a:A 2024-11-07T09:38:52,443 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T09:38:52,473 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:52,476 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411074352fca4e3cb40cc881125575669c274_e4cc6107007e595af5a0df7b5bc3711a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411074352fca4e3cb40cc881125575669c274_e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:52,476 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/4dfd8e9e89e74e85bff90fa9821e6067, store: [table=TestAcidGuarantees family=A region=e4cc6107007e595af5a0df7b5bc3711a] 2024-11-07T09:38:52,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/4dfd8e9e89e74e85bff90fa9821e6067 is 175, key is test_row_0/A:col10/1730972332063/Put/seqid=0 2024-11-07T09:38:52,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742466_1642 (size=22561) 2024-11-07T09:38:52,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-07T09:38:52,573 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:38:52,880 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=297, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/4dfd8e9e89e74e85bff90fa9821e6067 2024-11-07T09:38:52,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/3ab34670f4214a85977ad707607069dd is 50, key is test_row_0/B:col10/1730972332063/Put/seqid=0 2024-11-07T09:38:52,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742467_1643 (size=9857) 2024-11-07T09:38:53,289 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/3ab34670f4214a85977ad707607069dd 2024-11-07T09:38:53,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/2915283afb334771bfc9c54fc6d9fc49 is 50, key is test_row_0/C:col10/1730972332063/Put/seqid=0 2024-11-07T09:38:53,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742468_1644 (size=9857) 2024-11-07T09:38:53,697 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/2915283afb334771bfc9c54fc6d9fc49 2024-11-07T09:38:53,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/4dfd8e9e89e74e85bff90fa9821e6067 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/4dfd8e9e89e74e85bff90fa9821e6067 2024-11-07T09:38:53,703 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/4dfd8e9e89e74e85bff90fa9821e6067, entries=100, sequenceid=297, filesize=22.0 K 2024-11-07T09:38:53,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/3ab34670f4214a85977ad707607069dd as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/3ab34670f4214a85977ad707607069dd 2024-11-07T09:38:53,706 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/3ab34670f4214a85977ad707607069dd, entries=100, sequenceid=297, filesize=9.6 K 2024-11-07T09:38:53,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/2915283afb334771bfc9c54fc6d9fc49 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/2915283afb334771bfc9c54fc6d9fc49 2024-11-07T09:38:53,708 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/2915283afb334771bfc9c54fc6d9fc49, entries=100, sequenceid=297, filesize=9.6 K 2024-11-07T09:38:53,709 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=13.42 KB/13740 for e4cc6107007e595af5a0df7b5bc3711a in 1644ms, sequenceid=297, compaction requested=false 2024-11-07T09:38:53,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e4cc6107007e595af5a0df7b5bc3711a: 2024-11-07T09:38:55,084 INFO [Thread-2491 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2491. 2024-11-07T09:38:55,084 DEBUG [Thread-2491 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T09:38:55,084 DEBUG [Thread-2491 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:55,085 DEBUG [Thread-2491 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:55,085 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:38:55,085 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-07T09:38:55,085 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-11-07T09:38:55,085 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-07T09:38:55,085 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-07T09:38:55,085 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-07T09:38:55,085 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-07T09:38:55,085 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-07T09:38:55,085 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 779 2024-11-07T09:38:55,085 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 783 2024-11-07T09:38:55,085 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 758 2024-11-07T09:38:55,085 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 773 2024-11-07T09:38:55,085 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 771 2024-11-07T09:38:55,085 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-07T09:38:55,085 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T09:38:55,085 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T09:38:55,085 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:55,085 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:55,085 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:38:55,086 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-07T09:38:55,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-07T09:38:55,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-07T09:38:55,088 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730972335088"}]},"ts":"1730972335088"} 2024-11-07T09:38:55,089 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-07T09:38:55,089 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-07T09:38:55,090 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T09:38:55,091 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e4cc6107007e595af5a0df7b5bc3711a, UNASSIGN}] 2024-11-07T09:38:55,091 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e4cc6107007e595af5a0df7b5bc3711a, UNASSIGN 2024-11-07T09:38:55,092 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=e4cc6107007e595af5a0df7b5bc3711a, regionState=CLOSING, regionLocation=9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:55,093 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e4cc6107007e595af5a0df7b5bc3711a, UNASSIGN because future has completed 2024-11-07T09:38:55,093 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T09:38:55,093 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970}] 2024-11-07T09:38:55,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-07T09:38:55,245 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(122): Close e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:55,245 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T09:38:55,245 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1722): Closing e4cc6107007e595af5a0df7b5bc3711a, disabling compactions & flushes 2024-11-07T09:38:55,245 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:55,245 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:55,245 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. after waiting 0 ms 2024-11-07T09:38:55,245 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:55,245 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(2902): Flushing e4cc6107007e595af5a0df7b5bc3711a 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-07T09:38:55,250 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411078ae9d05b063c42a09e4e1ce8a1bf0d38_e4cc6107007e595af5a0df7b5bc3711a is 50, key is test_row_0/A:col10/1730972332074/Put/seqid=0 2024-11-07T09:38:55,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742469_1645 (size=9914) 2024-11-07T09:38:55,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-07T09:38:55,653 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T09:38:55,656 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411078ae9d05b063c42a09e4e1ce8a1bf0d38_e4cc6107007e595af5a0df7b5bc3711a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411078ae9d05b063c42a09e4e1ce8a1bf0d38_e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:55,656 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/4b90cb9e1e68459f9da4bc456fe934e6, store: [table=TestAcidGuarantees family=A region=e4cc6107007e595af5a0df7b5bc3711a] 2024-11-07T09:38:55,657 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/4b90cb9e1e68459f9da4bc456fe934e6 is 175, key is test_row_0/A:col10/1730972332074/Put/seqid=0 2024-11-07T09:38:55,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742470_1646 (size=22561) 2024-11-07T09:38:55,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-07T09:38:56,060 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=304, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/4b90cb9e1e68459f9da4bc456fe934e6 2024-11-07T09:38:56,065 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/f759bd80734242c797e5b181a68a8203 is 50, key is test_row_0/B:col10/1730972332074/Put/seqid=0 2024-11-07T09:38:56,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742471_1647 (size=9857) 2024-11-07T09:38:56,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-07T09:38:56,469 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/f759bd80734242c797e5b181a68a8203 2024-11-07T09:38:56,474 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/632cd79610bd498392ebc9ebec4ae94a is 50, key is test_row_0/C:col10/1730972332074/Put/seqid=0 2024-11-07T09:38:56,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742472_1648 (size=9857) 2024-11-07T09:38:56,672 DEBUG [regionserver/9f1eb259a29b:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.0, tune throughput to 50.00 MB/second 2024-11-07T09:38:56,877 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/632cd79610bd498392ebc9ebec4ae94a 2024-11-07T09:38:56,880 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/A/4b90cb9e1e68459f9da4bc456fe934e6 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/4b90cb9e1e68459f9da4bc456fe934e6 2024-11-07T09:38:56,883 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/4b90cb9e1e68459f9da4bc456fe934e6, entries=100, sequenceid=304, filesize=22.0 K 2024-11-07T09:38:56,884 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/B/f759bd80734242c797e5b181a68a8203 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/f759bd80734242c797e5b181a68a8203 2024-11-07T09:38:56,886 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/f759bd80734242c797e5b181a68a8203, entries=100, sequenceid=304, filesize=9.6 K 2024-11-07T09:38:56,887 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/.tmp/C/632cd79610bd498392ebc9ebec4ae94a as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/632cd79610bd498392ebc9ebec4ae94a 2024-11-07T09:38:56,889 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/632cd79610bd498392ebc9ebec4ae94a, entries=100, sequenceid=304, filesize=9.6 K 2024-11-07T09:38:56,890 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for e4cc6107007e595af5a0df7b5bc3711a in 1645ms, sequenceid=304, compaction requested=true 2024-11-07T09:38:56,890 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/fd119cfe8c25426898eec077d4333d79, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/c1fec97221aa440ebc6232e2873c8a9b, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/c8a8315ed9e74feaa4574aee7c748a3a, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/e4a4011bd6bd4b68a36748482538036b, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/f365ce54fe5e4cfcbb1858035e6f4a2a, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/91d6a1d77f4c460fb1fb7e072cbc156e, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/e65e7816006b46db8ad331ba51db7759, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/338b501de47e471b87678cf61758c18d, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/64ddd7511f1b4cf2b4a70860709208c3, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/f327137223704eb49e86df06f666903d, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/6d14692d71254e929fc7bdee946f952a, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/9f11bbb34a9e4565a95ea7328e93da11, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/6c7e40889c09437a8e4068195fa5910e, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/577623b0d50740d4bc469948bd92e915, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/09098679acd0404ca045d397ac0036f2, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/5d149ba83d034089ab21393fba0af980, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/6e45489eb78c4810bf6d210235f413ba, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/43a83c3ded31437087c01d85b22d16fc, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/eb2c4d17f6db433888714658185f78ca] to archive 2024-11-07T09:38:56,891 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T09:38:56,892 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/fd119cfe8c25426898eec077d4333d79 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/fd119cfe8c25426898eec077d4333d79 2024-11-07T09:38:56,893 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/c1fec97221aa440ebc6232e2873c8a9b to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/c1fec97221aa440ebc6232e2873c8a9b 2024-11-07T09:38:56,893 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/c8a8315ed9e74feaa4574aee7c748a3a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/c8a8315ed9e74feaa4574aee7c748a3a 2024-11-07T09:38:56,894 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/e4a4011bd6bd4b68a36748482538036b to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/e4a4011bd6bd4b68a36748482538036b 2024-11-07T09:38:56,895 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/f365ce54fe5e4cfcbb1858035e6f4a2a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/f365ce54fe5e4cfcbb1858035e6f4a2a 2024-11-07T09:38:56,895 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/91d6a1d77f4c460fb1fb7e072cbc156e to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/91d6a1d77f4c460fb1fb7e072cbc156e 2024-11-07T09:38:56,896 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/e65e7816006b46db8ad331ba51db7759 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/e65e7816006b46db8ad331ba51db7759 2024-11-07T09:38:56,897 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/338b501de47e471b87678cf61758c18d to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/338b501de47e471b87678cf61758c18d 2024-11-07T09:38:56,897 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/64ddd7511f1b4cf2b4a70860709208c3 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/64ddd7511f1b4cf2b4a70860709208c3 2024-11-07T09:38:56,898 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/f327137223704eb49e86df06f666903d to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/f327137223704eb49e86df06f666903d 2024-11-07T09:38:56,899 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/6d14692d71254e929fc7bdee946f952a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/6d14692d71254e929fc7bdee946f952a 2024-11-07T09:38:56,899 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/9f11bbb34a9e4565a95ea7328e93da11 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/9f11bbb34a9e4565a95ea7328e93da11 2024-11-07T09:38:56,900 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/6c7e40889c09437a8e4068195fa5910e to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/6c7e40889c09437a8e4068195fa5910e 2024-11-07T09:38:56,900 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/577623b0d50740d4bc469948bd92e915 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/577623b0d50740d4bc469948bd92e915 2024-11-07T09:38:56,901 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/09098679acd0404ca045d397ac0036f2 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/09098679acd0404ca045d397ac0036f2 2024-11-07T09:38:56,902 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/5d149ba83d034089ab21393fba0af980 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/5d149ba83d034089ab21393fba0af980 2024-11-07T09:38:56,902 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/6e45489eb78c4810bf6d210235f413ba to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/6e45489eb78c4810bf6d210235f413ba 2024-11-07T09:38:56,903 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/43a83c3ded31437087c01d85b22d16fc to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/43a83c3ded31437087c01d85b22d16fc 2024-11-07T09:38:56,904 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/eb2c4d17f6db433888714658185f78ca to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/eb2c4d17f6db433888714658185f78ca 2024-11-07T09:38:56,905 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/fc5577510e4647c0928888199b81a1ff, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/291a2f21ad134074b499724083d3ba1c, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/2b2ec2c512cd4fad9e5356a6a9598dc1, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/5b1b0ef1847149f9adafea9521a14093, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/689605facccf4d61a10cb02b8f0387b0, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/2bcf8afb2a924e66b9eaa6599f125a69, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/8d3747be76e64e1b99924cef6424d4bd, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/7e2c6397ba104b2fb4c6819a1851071f, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/245209913f4342c482edd81a18001e21, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/ef0910e7bd9840838d641dbd982c1111, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/9058eef2ede4427da2c4de91bdac369c, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/04bd274447bb490c937f47c810c4d086, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/5e043fa519d441549e1d8741cc3e49a2, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/dcb0b445b9ac4d51a4903cc1e290ae32, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/2155445caa6b44ab804b6d35f57e1c4f, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/b1f85386ea92425ebbf974ad9ba07962, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/ac8a88df983f43d6943705cee439911e, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/6743acf1504444e6b6beb94b56b67e4c, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/edccb37e73894a9cb78a30f9df182937] to archive 2024-11-07T09:38:56,905 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T09:38:56,906 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/fc5577510e4647c0928888199b81a1ff to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/fc5577510e4647c0928888199b81a1ff 2024-11-07T09:38:56,907 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/291a2f21ad134074b499724083d3ba1c to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/291a2f21ad134074b499724083d3ba1c 2024-11-07T09:38:56,908 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/2b2ec2c512cd4fad9e5356a6a9598dc1 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/2b2ec2c512cd4fad9e5356a6a9598dc1 2024-11-07T09:38:56,908 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/5b1b0ef1847149f9adafea9521a14093 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/5b1b0ef1847149f9adafea9521a14093 2024-11-07T09:38:56,909 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/689605facccf4d61a10cb02b8f0387b0 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/689605facccf4d61a10cb02b8f0387b0 2024-11-07T09:38:56,910 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/2bcf8afb2a924e66b9eaa6599f125a69 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/2bcf8afb2a924e66b9eaa6599f125a69 2024-11-07T09:38:56,910 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/8d3747be76e64e1b99924cef6424d4bd to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/8d3747be76e64e1b99924cef6424d4bd 2024-11-07T09:38:56,911 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/7e2c6397ba104b2fb4c6819a1851071f to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/7e2c6397ba104b2fb4c6819a1851071f 2024-11-07T09:38:56,912 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/245209913f4342c482edd81a18001e21 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/245209913f4342c482edd81a18001e21 2024-11-07T09:38:56,912 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/ef0910e7bd9840838d641dbd982c1111 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/ef0910e7bd9840838d641dbd982c1111 2024-11-07T09:38:56,913 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/9058eef2ede4427da2c4de91bdac369c to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/9058eef2ede4427da2c4de91bdac369c 2024-11-07T09:38:56,914 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/04bd274447bb490c937f47c810c4d086 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/04bd274447bb490c937f47c810c4d086 2024-11-07T09:38:56,915 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/5e043fa519d441549e1d8741cc3e49a2 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/5e043fa519d441549e1d8741cc3e49a2 2024-11-07T09:38:56,915 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/dcb0b445b9ac4d51a4903cc1e290ae32 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/dcb0b445b9ac4d51a4903cc1e290ae32 2024-11-07T09:38:56,916 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/2155445caa6b44ab804b6d35f57e1c4f to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/2155445caa6b44ab804b6d35f57e1c4f 2024-11-07T09:38:56,917 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/b1f85386ea92425ebbf974ad9ba07962 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/b1f85386ea92425ebbf974ad9ba07962 2024-11-07T09:38:56,917 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/ac8a88df983f43d6943705cee439911e to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/ac8a88df983f43d6943705cee439911e 2024-11-07T09:38:56,918 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/6743acf1504444e6b6beb94b56b67e4c to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/6743acf1504444e6b6beb94b56b67e4c 2024-11-07T09:38:56,919 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/edccb37e73894a9cb78a30f9df182937 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/edccb37e73894a9cb78a30f9df182937 2024-11-07T09:38:56,920 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/6acd9fea38984b73a9548e4e53c87daf, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/e4cabc2f6dae4d89b361766db0ac2c87, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/d417bdb1c21a4660aeadd751c7fece59, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/c1ffa951dc574482b98aeac5f50d30f8, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/2c78ced2ba9841bab8f0abf60b027135, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/a2b1271857f143acb62c54f48be29199, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/3e38a4c9bb1f4a9f903252aa59e0320a, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/85546a63f8034046975fde0847299591, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/0d11bda8fdeb4c3c9e5a4a18df5ca7c4, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/92ece59a5a3745c79b39542e651799ed, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/c85ac8583e414a9784ae399a5f441d9a, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/b3629c9fc1594fc8a404e28e04f41d8d, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/1c8a8ff222304f5eb5183a796551f815, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/33ffc27a03b04fbc9f516bb4c460cc9d, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/2a64c9dea9564cb0acd3e5af3c1ba7ba, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/46eedc2f70c64a19833be0ca75c1a921, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/bfec82264ccb40edb292cc759371a437, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/16c1e08343bb487babbe80aa7777e981, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/809c3cc3d62d4cc7ac25b7fc27a2dcc0] to archive 2024-11-07T09:38:56,920 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T09:38:56,921 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/6acd9fea38984b73a9548e4e53c87daf to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/6acd9fea38984b73a9548e4e53c87daf 2024-11-07T09:38:56,922 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/e4cabc2f6dae4d89b361766db0ac2c87 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/e4cabc2f6dae4d89b361766db0ac2c87 2024-11-07T09:38:56,923 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/d417bdb1c21a4660aeadd751c7fece59 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/d417bdb1c21a4660aeadd751c7fece59 2024-11-07T09:38:56,923 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/c1ffa951dc574482b98aeac5f50d30f8 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/c1ffa951dc574482b98aeac5f50d30f8 2024-11-07T09:38:56,924 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/2c78ced2ba9841bab8f0abf60b027135 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/2c78ced2ba9841bab8f0abf60b027135 2024-11-07T09:38:56,925 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/a2b1271857f143acb62c54f48be29199 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/a2b1271857f143acb62c54f48be29199 2024-11-07T09:38:56,926 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/3e38a4c9bb1f4a9f903252aa59e0320a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/3e38a4c9bb1f4a9f903252aa59e0320a 2024-11-07T09:38:56,926 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/85546a63f8034046975fde0847299591 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/85546a63f8034046975fde0847299591 2024-11-07T09:38:56,927 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/0d11bda8fdeb4c3c9e5a4a18df5ca7c4 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/0d11bda8fdeb4c3c9e5a4a18df5ca7c4 2024-11-07T09:38:56,927 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/92ece59a5a3745c79b39542e651799ed to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/92ece59a5a3745c79b39542e651799ed 2024-11-07T09:38:56,928 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/c85ac8583e414a9784ae399a5f441d9a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/c85ac8583e414a9784ae399a5f441d9a 2024-11-07T09:38:56,929 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/b3629c9fc1594fc8a404e28e04f41d8d to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/b3629c9fc1594fc8a404e28e04f41d8d 2024-11-07T09:38:56,930 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/1c8a8ff222304f5eb5183a796551f815 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/1c8a8ff222304f5eb5183a796551f815 2024-11-07T09:38:56,930 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/33ffc27a03b04fbc9f516bb4c460cc9d to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/33ffc27a03b04fbc9f516bb4c460cc9d 2024-11-07T09:38:56,931 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/2a64c9dea9564cb0acd3e5af3c1ba7ba to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/2a64c9dea9564cb0acd3e5af3c1ba7ba 2024-11-07T09:38:56,932 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/46eedc2f70c64a19833be0ca75c1a921 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/46eedc2f70c64a19833be0ca75c1a921 2024-11-07T09:38:56,932 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/bfec82264ccb40edb292cc759371a437 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/bfec82264ccb40edb292cc759371a437 2024-11-07T09:38:56,933 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/16c1e08343bb487babbe80aa7777e981 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/16c1e08343bb487babbe80aa7777e981 2024-11-07T09:38:56,934 DEBUG [StoreCloser-TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/809c3cc3d62d4cc7ac25b7fc27a2dcc0 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/809c3cc3d62d4cc7ac25b7fc27a2dcc0 2024-11-07T09:38:56,937 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/recovered.edits/307.seqid, newMaxSeqId=307, maxSeqId=4 2024-11-07T09:38:56,937 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. 2024-11-07T09:38:56,937 DEBUG [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1676): Region close journal for e4cc6107007e595af5a0df7b5bc3711a: Waiting for close lock at 1730972335245Running coprocessor pre-close hooks at 1730972335245Disabling compacts and flushes for region at 1730972335245Disabling writes for close at 1730972335245Obtaining lock to block concurrent updates at 1730972335245Preparing flush snapshotting stores in e4cc6107007e595af5a0df7b5bc3711a at 1730972335245Finished memstore snapshotting TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1730972335246 (+1 ms)Flushing stores of TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a. at 1730972335246Flushing e4cc6107007e595af5a0df7b5bc3711a/A: creating writer at 1730972335246Flushing mob file e4cc6107007e595af5a0df7b5bc3711a/A: appending metadata at 1730972335249 (+3 ms)Flushing mob file e4cc6107007e595af5a0df7b5bc3711a/A: closing flushed file at 1730972335249Flushing e4cc6107007e595af5a0df7b5bc3711a/A: appending metadata at 1730972335656 (+407 ms)Flushing e4cc6107007e595af5a0df7b5bc3711a/A: closing flushed file at 1730972335657 (+1 ms)Flushing e4cc6107007e595af5a0df7b5bc3711a/B: creating writer at 1730972336063 (+406 ms)Flushing e4cc6107007e595af5a0df7b5bc3711a/B: appending metadata at 1730972336065 (+2 ms)Flushing e4cc6107007e595af5a0df7b5bc3711a/B: closing flushed file at 1730972336065Flushing e4cc6107007e595af5a0df7b5bc3711a/C: creating writer at 1730972336472 (+407 ms)Flushing e4cc6107007e595af5a0df7b5bc3711a/C: appending metadata at 1730972336473 (+1 ms)Flushing e4cc6107007e595af5a0df7b5bc3711a/C: closing flushed file at 1730972336473Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c547b4d: reopening flushed file at 1730972336880 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@43b52f75: reopening flushed file at 1730972336883 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d6786c6: reopening flushed file at 1730972336886 (+3 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for e4cc6107007e595af5a0df7b5bc3711a in 1645ms, sequenceid=304, compaction requested=true at 1730972336890 (+4 ms)Writing region close event to WAL at 1730972336935 (+45 ms)Running coprocessor post-close hooks at 1730972336937 (+2 ms)Closed at 1730972336937 2024-11-07T09:38:56,939 INFO [RS_CLOSE_REGION-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(157): Closed e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:56,939 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=e4cc6107007e595af5a0df7b5bc3711a, regionState=CLOSED 2024-11-07T09:38:56,940 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 because future has completed 2024-11-07T09:38:56,942 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-07T09:38:56,942 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseRegionProcedure e4cc6107007e595af5a0df7b5bc3711a, server=9f1eb259a29b,42895,1730972153970 in 1.8480 sec 2024-11-07T09:38:56,944 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=164, resume processing ppid=163 2024-11-07T09:38:56,944 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, ppid=163, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e4cc6107007e595af5a0df7b5bc3711a, UNASSIGN in 1.8510 sec 2024-11-07T09:38:56,945 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-07T09:38:56,945 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8540 sec 2024-11-07T09:38:56,947 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730972336946"}]},"ts":"1730972336946"} 2024-11-07T09:38:56,948 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-07T09:38:56,948 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-07T09:38:56,949 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8620 sec 2024-11-07T09:38:57,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-07T09:38:57,233 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:38:57,234 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-07T09:38:57,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T09:38:57,235 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T09:38:57,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-07T09:38:57,236 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=166, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T09:38:57,237 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:57,239 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A, FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B, FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C, FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/recovered.edits] 2024-11-07T09:38:57,241 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/4b90cb9e1e68459f9da4bc456fe934e6 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/4b90cb9e1e68459f9da4bc456fe934e6 2024-11-07T09:38:57,241 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/4dfd8e9e89e74e85bff90fa9821e6067 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/4dfd8e9e89e74e85bff90fa9821e6067 2024-11-07T09:38:57,242 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/f1848c6a168c4f4fbc6efe33989df4f4 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/A/f1848c6a168c4f4fbc6efe33989df4f4 2024-11-07T09:38:57,243 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/1196bd662b8b4445ac79da797e454a59 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/1196bd662b8b4445ac79da797e454a59 2024-11-07T09:38:57,244 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/3ab34670f4214a85977ad707607069dd to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/3ab34670f4214a85977ad707607069dd 2024-11-07T09:38:57,245 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/f759bd80734242c797e5b181a68a8203 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/B/f759bd80734242c797e5b181a68a8203 2024-11-07T09:38:57,246 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/2915283afb334771bfc9c54fc6d9fc49 to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/2915283afb334771bfc9c54fc6d9fc49 2024-11-07T09:38:57,247 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/632cd79610bd498392ebc9ebec4ae94a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/632cd79610bd498392ebc9ebec4ae94a 2024-11-07T09:38:57,248 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/efbbe8912c944c6fb82c8005e8bfc0bb to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/C/efbbe8912c944c6fb82c8005e8bfc0bb 2024-11-07T09:38:57,249 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/recovered.edits/307.seqid to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a/recovered.edits/307.seqid 2024-11-07T09:38:57,250 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/default/TestAcidGuarantees/e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:57,250 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-07T09:38:57,250 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-07T09:38:57,250 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-07T09:38:57,252 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110724dcd2198fa947a0aaa7aac282e7217d_e4cc6107007e595af5a0df7b5bc3711a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110724dcd2198fa947a0aaa7aac282e7217d_e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:57,253 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411073b0853898dce4277b033319f6e050573_e4cc6107007e595af5a0df7b5bc3711a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411073b0853898dce4277b033319f6e050573_e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:57,254 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411074352fca4e3cb40cc881125575669c274_e4cc6107007e595af5a0df7b5bc3711a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411074352fca4e3cb40cc881125575669c274_e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:57,254 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411075681419ffe454b9d9901205dc908e875_e4cc6107007e595af5a0df7b5bc3711a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411075681419ffe454b9d9901205dc908e875_e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:57,255 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110757cf1ea4cc6f4876845515006bb5837a_e4cc6107007e595af5a0df7b5bc3711a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110757cf1ea4cc6f4876845515006bb5837a_e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:57,256 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411076f0628fe31ef4035b847c5ed9fffc8e4_e4cc6107007e595af5a0df7b5bc3711a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411076f0628fe31ef4035b847c5ed9fffc8e4_e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:57,256 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411077ffc366721944c9aae56e2b3367cfeb1_e4cc6107007e595af5a0df7b5bc3711a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411077ffc366721944c9aae56e2b3367cfeb1_e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:57,257 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411078ae9d05b063c42a09e4e1ce8a1bf0d38_e4cc6107007e595af5a0df7b5bc3711a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411078ae9d05b063c42a09e4e1ce8a1bf0d38_e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:57,258 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411078c303b6ceca34e089e38c059f233dbb1_e4cc6107007e595af5a0df7b5bc3711a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411078c303b6ceca34e089e38c059f233dbb1_e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:57,259 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107a21fd4b9fdd54beb86924f89edf7b2d9_e4cc6107007e595af5a0df7b5bc3711a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107a21fd4b9fdd54beb86924f89edf7b2d9_e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:57,259 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107aa3c85f41c4c45df93b60757e862cb59_e4cc6107007e595af5a0df7b5bc3711a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107aa3c85f41c4c45df93b60757e862cb59_e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:57,260 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107b07575debecd4df1a906f4c1604a0110_e4cc6107007e595af5a0df7b5bc3711a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107b07575debecd4df1a906f4c1604a0110_e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:57,261 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107c5e2f4891cea4582a955cc197d021dbb_e4cc6107007e595af5a0df7b5bc3711a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107c5e2f4891cea4582a955cc197d021dbb_e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:57,261 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107db1619780e454839a3eaf863cdcd99b3_e4cc6107007e595af5a0df7b5bc3711a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107db1619780e454839a3eaf863cdcd99b3_e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:57,262 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107e95db33f3b814e99a99b52e9aab077f4_e4cc6107007e595af5a0df7b5bc3711a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107e95db33f3b814e99a99b52e9aab077f4_e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:57,263 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107ffb48b84a0da4370a463a72e3de00262_e4cc6107007e595af5a0df7b5bc3711a to hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107ffb48b84a0da4370a463a72e3de00262_e4cc6107007e595af5a0df7b5bc3711a 2024-11-07T09:38:57,263 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-07T09:38:57,265 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=166, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T09:38:57,266 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-07T09:38:57,268 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-07T09:38:57,269 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=166, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T09:38:57,269 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-07T09:38:57,269 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1730972337269"}]},"ts":"9223372036854775807"} 2024-11-07T09:38:57,270 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-07T09:38:57,270 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e4cc6107007e595af5a0df7b5bc3711a, NAME => 'TestAcidGuarantees,,1730972309292.e4cc6107007e595af5a0df7b5bc3711a.', STARTKEY => '', ENDKEY => ''}] 2024-11-07T09:38:57,270 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-07T09:38:57,270 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1730972337270"}]},"ts":"9223372036854775807"} 2024-11-07T09:38:57,271 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-07T09:38:57,272 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=166, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T09:38:57,273 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 38 msec 2024-11-07T09:38:57,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-07T09:38:57,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-07T09:38:57,343 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-07T09:38:57,352 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=241 (was 240) - Thread LEAK? -, OpenFileDescriptor=459 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=2334 (was 1094) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4263 (was 4280) 2024-11-07T09:38:57,352 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-07T09:38:57,352 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T09:38:57,352 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T09:38:57,352 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:57,352 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:57,352 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T09:38:57,352 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-07T09:38:57,352 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=900566320, stopped=false 2024-11-07T09:38:57,353 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=9f1eb259a29b,41689,1730972153310 2024-11-07T09:38:57,355 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41689-0x100f1c8dd440000, quorum=127.0.0.1:59189, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-07T09:38:57,355 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42895-0x100f1c8dd440001, quorum=127.0.0.1:59189, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-07T09:38:57,355 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42895-0x100f1c8dd440001, quorum=127.0.0.1:59189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T09:38:57,355 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41689-0x100f1c8dd440000, quorum=127.0.0.1:59189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T09:38:57,355 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T09:38:57,355 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T09:38:57,355 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T09:38:57,355 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:57,355 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42895-0x100f1c8dd440001, quorum=127.0.0.1:59189, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T09:38:57,355 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41689-0x100f1c8dd440000, quorum=127.0.0.1:59189, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T09:38:57,356 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9f1eb259a29b,42895,1730972153970' ***** 2024-11-07T09:38:57,356 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-07T09:38:57,356 INFO [RS:0;9f1eb259a29b:42895 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-07T09:38:57,356 INFO [RS:0;9f1eb259a29b:42895 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-07T09:38:57,357 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-07T09:38:57,357 INFO [RS:0;9f1eb259a29b:42895 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-07T09:38:57,357 INFO [RS:0;9f1eb259a29b:42895 {}] regionserver.HRegionServer(959): stopping server 9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:57,357 INFO [RS:0;9f1eb259a29b:42895 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-07T09:38:57,357 INFO [RS:0;9f1eb259a29b:42895 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;9f1eb259a29b:42895. 2024-11-07T09:38:57,357 DEBUG [RS:0;9f1eb259a29b:42895 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T09:38:57,357 DEBUG [RS:0;9f1eb259a29b:42895 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:57,357 INFO [RS:0;9f1eb259a29b:42895 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-07T09:38:57,357 INFO [RS:0;9f1eb259a29b:42895 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-07T09:38:57,357 INFO [RS:0;9f1eb259a29b:42895 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-07T09:38:57,357 INFO [RS:0;9f1eb259a29b:42895 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-07T09:38:57,358 INFO [RS:0;9f1eb259a29b:42895 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-07T09:38:57,358 DEBUG [RS:0;9f1eb259a29b:42895 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-07T09:38:57,358 DEBUG [RS:0;9f1eb259a29b:42895 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T09:38:57,358 DEBUG [RS_CLOSE_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-07T09:38:57,358 INFO [RS_CLOSE_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-07T09:38:57,358 DEBUG [RS_CLOSE_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-07T09:38:57,358 DEBUG [RS_CLOSE_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-07T09:38:57,358 DEBUG [RS_CLOSE_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-07T09:38:57,358 INFO [RS_CLOSE_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-07T09:38:57,381 DEBUG [RS_CLOSE_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/hbase/meta/1588230740/.tmp/info/2f9d49a139e54ecab3d00e09207f6a83 is 95, key is TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26./info:/1730972185925/DeleteFamily/seqid=0 2024-11-07T09:38:57,383 INFO [regionserver/9f1eb259a29b:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T09:38:57,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742473_1649 (size=6514) 2024-11-07T09:38:57,558 DEBUG [RS:0;9f1eb259a29b:42895 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T09:38:57,631 INFO [regionserver/9f1eb259a29b:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-07T09:38:57,631 INFO [regionserver/9f1eb259a29b:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-07T09:38:57,758 DEBUG [RS:0;9f1eb259a29b:42895 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T09:38:57,784 INFO [RS_CLOSE_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/hbase/meta/1588230740/.tmp/info/2f9d49a139e54ecab3d00e09207f6a83 2024-11-07T09:38:57,802 DEBUG [RS_CLOSE_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/hbase/meta/1588230740/.tmp/ns/c848076099cc4bfd8e689b515cfd84c9 is 93, key is TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26./ns:/1730972185879/DeleteFamily/seqid=0 2024-11-07T09:38:57,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742474_1650 (size=5906) 2024-11-07T09:38:57,959 DEBUG [RS:0;9f1eb259a29b:42895 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T09:38:58,159 DEBUG [RS:0;9f1eb259a29b:42895 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T09:38:58,206 INFO [RS_CLOSE_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/hbase/meta/1588230740/.tmp/ns/c848076099cc4bfd8e689b515cfd84c9 2024-11-07T09:38:58,223 DEBUG [RS_CLOSE_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/hbase/meta/1588230740/.tmp/rep_barrier/f5ff6489bf3f4d59993badbb1c171388 is 102, key is TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26./rep_barrier:/1730972185879/DeleteFamily/seqid=0 2024-11-07T09:38:58,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742475_1651 (size=6025) 2024-11-07T09:38:58,359 DEBUG [RS:0;9f1eb259a29b:42895 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T09:38:58,559 DEBUG [RS:0;9f1eb259a29b:42895 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T09:38:58,626 INFO [RS_CLOSE_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/hbase/meta/1588230740/.tmp/rep_barrier/f5ff6489bf3f4d59993badbb1c171388 2024-11-07T09:38:58,643 DEBUG [RS_CLOSE_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/hbase/meta/1588230740/.tmp/table/88a1a1ddf04b4ae19884fb5e10a0baf6 is 96, key is TestAcidGuarantees,,1730972158185.116dccc5098610cd38ab507cb3daad26./table:/1730972185879/DeleteFamily/seqid=0 2024-11-07T09:38:58,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742476_1652 (size=5936) 2024-11-07T09:38:58,759 DEBUG [RS:0;9f1eb259a29b:42895 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T09:38:58,960 DEBUG [RS:0;9f1eb259a29b:42895 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T09:38:59,046 INFO [RS_CLOSE_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/hbase/meta/1588230740/.tmp/table/88a1a1ddf04b4ae19884fb5e10a0baf6 2024-11-07T09:38:59,050 DEBUG [RS_CLOSE_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/hbase/meta/1588230740/.tmp/info/2f9d49a139e54ecab3d00e09207f6a83 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/hbase/meta/1588230740/info/2f9d49a139e54ecab3d00e09207f6a83 2024-11-07T09:38:59,052 INFO [RS_CLOSE_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/hbase/meta/1588230740/info/2f9d49a139e54ecab3d00e09207f6a83, entries=12, sequenceid=90, filesize=6.4 K 2024-11-07T09:38:59,053 DEBUG [RS_CLOSE_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/hbase/meta/1588230740/.tmp/ns/c848076099cc4bfd8e689b515cfd84c9 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/hbase/meta/1588230740/ns/c848076099cc4bfd8e689b515cfd84c9 2024-11-07T09:38:59,055 INFO [RS_CLOSE_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/hbase/meta/1588230740/ns/c848076099cc4bfd8e689b515cfd84c9, entries=8, sequenceid=90, filesize=5.8 K 2024-11-07T09:38:59,056 DEBUG [RS_CLOSE_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/hbase/meta/1588230740/.tmp/rep_barrier/f5ff6489bf3f4d59993badbb1c171388 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/hbase/meta/1588230740/rep_barrier/f5ff6489bf3f4d59993badbb1c171388 2024-11-07T09:38:59,058 INFO [RS_CLOSE_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/hbase/meta/1588230740/rep_barrier/f5ff6489bf3f4d59993badbb1c171388, entries=6, sequenceid=90, filesize=5.9 K 2024-11-07T09:38:59,059 DEBUG [RS_CLOSE_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/hbase/meta/1588230740/.tmp/table/88a1a1ddf04b4ae19884fb5e10a0baf6 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/hbase/meta/1588230740/table/88a1a1ddf04b4ae19884fb5e10a0baf6 2024-11-07T09:38:59,061 INFO [RS_CLOSE_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/hbase/meta/1588230740/table/88a1a1ddf04b4ae19884fb5e10a0baf6, entries=7, sequenceid=90, filesize=5.8 K 2024-11-07T09:38:59,061 INFO [RS_CLOSE_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1703ms, sequenceid=90, compaction requested=false 2024-11-07T09:38:59,066 DEBUG [RS_CLOSE_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-07T09:38:59,066 DEBUG [RS_CLOSE_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-07T09:38:59,066 INFO [RS_CLOSE_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-07T09:38:59,066 DEBUG [RS_CLOSE_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1730972337358Running coprocessor pre-close hooks at 1730972337358Disabling compacts and flushes for region at 1730972337358Disabling writes for close at 1730972337358Obtaining lock to block concurrent updates at 1730972337358Preparing flush snapshotting stores in 1588230740 at 1730972337358Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1730972337358Flushing stores of hbase:meta,,1.1588230740 at 1730972337359 (+1 ms)Flushing 1588230740/info: creating writer at 1730972337359Flushing 1588230740/info: appending metadata at 1730972337380 (+21 ms)Flushing 1588230740/info: closing flushed file at 1730972337380Flushing 1588230740/ns: creating writer at 1730972337788 (+408 ms)Flushing 1588230740/ns: appending metadata at 1730972337802 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1730972337802Flushing 1588230740/rep_barrier: creating writer at 1730972338209 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1730972338223 (+14 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1730972338223Flushing 1588230740/table: creating writer at 1730972338629 (+406 ms)Flushing 1588230740/table: appending metadata at 1730972338643 (+14 ms)Flushing 1588230740/table: closing flushed file at 1730972338643Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d87f225: reopening flushed file at 1730972339049 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69c6faeb: reopening flushed file at 1730972339053 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3a282d43: reopening flushed file at 1730972339056 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@667222d7: reopening flushed file at 1730972339058 (+2 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1703ms, sequenceid=90, compaction requested=false at 1730972339062 (+4 ms)Writing region close event to WAL at 1730972339064 (+2 ms)Running coprocessor post-close hooks at 1730972339066 (+2 ms)Closed at 1730972339066 2024-11-07T09:38:59,067 DEBUG [RS_CLOSE_META-regionserver/9f1eb259a29b:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-07T09:38:59,160 INFO [RS:0;9f1eb259a29b:42895 {}] regionserver.HRegionServer(976): stopping server 9f1eb259a29b,42895,1730972153970; all regions closed. 2024-11-07T09:38:59,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741834_1010 (size=25344) 2024-11-07T09:38:59,166 DEBUG [RS:0;9f1eb259a29b:42895 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/oldWALs 2024-11-07T09:38:59,166 INFO [RS:0;9f1eb259a29b:42895 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9f1eb259a29b%2C42895%2C1730972153970.meta:.meta(num 1730972157796) 2024-11-07T09:38:59,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741832_1008 (size=15322178) 2024-11-07T09:38:59,169 DEBUG [RS:0;9f1eb259a29b:42895 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/oldWALs 2024-11-07T09:38:59,169 INFO [RS:0;9f1eb259a29b:42895 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9f1eb259a29b%2C42895%2C1730972153970:(num 1730972156787) 2024-11-07T09:38:59,169 DEBUG [RS:0;9f1eb259a29b:42895 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T09:38:59,169 INFO [RS:0;9f1eb259a29b:42895 {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T09:38:59,169 INFO [RS:0;9f1eb259a29b:42895 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-07T09:38:59,169 INFO [RS:0;9f1eb259a29b:42895 {}] hbase.ChoreService(370): Chore service for: regionserver/9f1eb259a29b:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-07T09:38:59,169 INFO [RS:0;9f1eb259a29b:42895 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-07T09:38:59,170 INFO [regionserver/9f1eb259a29b:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-07T09:38:59,170 INFO [RS:0;9f1eb259a29b:42895 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42895 2024-11-07T09:38:59,174 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42895-0x100f1c8dd440001, quorum=127.0.0.1:59189, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9f1eb259a29b,42895,1730972153970 2024-11-07T09:38:59,174 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41689-0x100f1c8dd440000, quorum=127.0.0.1:59189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-07T09:38:59,174 INFO [RS:0;9f1eb259a29b:42895 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-07T09:38:59,177 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9f1eb259a29b,42895,1730972153970] 2024-11-07T09:38:59,178 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9f1eb259a29b,42895,1730972153970 already deleted, retry=false 2024-11-07T09:38:59,178 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9f1eb259a29b,42895,1730972153970 expired; onlineServers=0 2024-11-07T09:38:59,178 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '9f1eb259a29b,41689,1730972153310' ***** 2024-11-07T09:38:59,178 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-07T09:38:59,178 INFO [M:0;9f1eb259a29b:41689 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-07T09:38:59,178 INFO [M:0;9f1eb259a29b:41689 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-07T09:38:59,178 DEBUG [M:0;9f1eb259a29b:41689 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-07T09:38:59,179 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-07T09:38:59,179 DEBUG [M:0;9f1eb259a29b:41689 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-07T09:38:59,179 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster-HFileCleaner.small.0-1730972156544 {}] cleaner.HFileCleaner(306): Exit Thread[master/9f1eb259a29b:0:becomeActiveMaster-HFileCleaner.small.0-1730972156544,5,FailOnTimeoutGroup] 2024-11-07T09:38:59,179 DEBUG [master/9f1eb259a29b:0:becomeActiveMaster-HFileCleaner.large.0-1730972156543 {}] cleaner.HFileCleaner(306): Exit Thread[master/9f1eb259a29b:0:becomeActiveMaster-HFileCleaner.large.0-1730972156543,5,FailOnTimeoutGroup] 2024-11-07T09:38:59,179 INFO [M:0;9f1eb259a29b:41689 {}] hbase.ChoreService(370): Chore service for: master/9f1eb259a29b:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-07T09:38:59,179 INFO [M:0;9f1eb259a29b:41689 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-07T09:38:59,179 DEBUG [M:0;9f1eb259a29b:41689 {}] master.HMaster(1795): Stopping service threads 2024-11-07T09:38:59,179 INFO [M:0;9f1eb259a29b:41689 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-07T09:38:59,179 INFO [M:0;9f1eb259a29b:41689 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T09:38:59,179 ERROR [M:0;9f1eb259a29b:41689 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-07T09:38:59,180 INFO [M:0;9f1eb259a29b:41689 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T09:38:59,180 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-07T09:38:59,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41689-0x100f1c8dd440000, quorum=127.0.0.1:59189, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-07T09:38:59,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41689-0x100f1c8dd440000, quorum=127.0.0.1:59189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T09:38:59,181 DEBUG [M:0;9f1eb259a29b:41689 {}] zookeeper.ZKUtil(347): master:41689-0x100f1c8dd440000, quorum=127.0.0.1:59189, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-07T09:38:59,181 WARN [M:0;9f1eb259a29b:41689 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-07T09:38:59,181 INFO [M:0;9f1eb259a29b:41689 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/.lastflushedseqids 2024-11-07T09:38:59,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742477_1653 (size=72) 2024-11-07T09:38:59,277 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42895-0x100f1c8dd440001, quorum=127.0.0.1:59189, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T09:38:59,277 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42895-0x100f1c8dd440001, quorum=127.0.0.1:59189, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T09:38:59,277 INFO [RS:0;9f1eb259a29b:42895 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-07T09:38:59,277 INFO [RS:0;9f1eb259a29b:42895 {}] regionserver.HRegionServer(1031): Exiting; stopping=9f1eb259a29b,42895,1730972153970; zookeeper connection closed. 2024-11-07T09:38:59,278 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6b3f498 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6b3f498 2024-11-07T09:38:59,278 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-07T09:38:59,590 INFO [M:0;9f1eb259a29b:41689 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T09:38:59,591 INFO [M:0;9f1eb259a29b:41689 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-07T09:38:59,591 DEBUG [M:0;9f1eb259a29b:41689 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-07T09:38:59,591 INFO [M:0;9f1eb259a29b:41689 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T09:38:59,591 DEBUG [M:0;9f1eb259a29b:41689 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T09:38:59,591 DEBUG [M:0;9f1eb259a29b:41689 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-07T09:38:59,591 DEBUG [M:0;9f1eb259a29b:41689 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T09:38:59,591 INFO [M:0;9f1eb259a29b:41689 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=750.27 KB heapSize=921.86 KB 2024-11-07T09:38:59,606 DEBUG [M:0;9f1eb259a29b:41689 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4cd7e0390398453296232501961ad367 is 82, key is hbase:meta,,1/info:regioninfo/1730972157879/Put/seqid=0 2024-11-07T09:38:59,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742478_1654 (size=5672) 2024-11-07T09:39:00,009 INFO [M:0;9f1eb259a29b:41689 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2123 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4cd7e0390398453296232501961ad367 2024-11-07T09:39:00,029 DEBUG [M:0;9f1eb259a29b:41689 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/614df009066d46e29b8940e213b5ce7d is 2276, key is \x00\x00\x00\x00\x00\x00\x00\x97/proc:d/1730972311327/Put/seqid=0 2024-11-07T09:39:00,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742479_1655 (size=43422) 2024-11-07T09:39:00,435 INFO [M:0;9f1eb259a29b:41689 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=749.72 KB at sequenceid=2123 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/614df009066d46e29b8940e213b5ce7d 2024-11-07T09:39:00,438 INFO [M:0;9f1eb259a29b:41689 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 614df009066d46e29b8940e213b5ce7d 2024-11-07T09:39:00,452 DEBUG [M:0;9f1eb259a29b:41689 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/481e2383f1404caf8f82fd8bc5760c3d is 69, key is 9f1eb259a29b,42895,1730972153970/rs:state/1730972156554/Put/seqid=0 2024-11-07T09:39:00,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073742480_1656 (size=5156) 2024-11-07T09:39:00,856 INFO [M:0;9f1eb259a29b:41689 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2123 (bloomFilter=true), to=hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/481e2383f1404caf8f82fd8bc5760c3d 2024-11-07T09:39:00,859 DEBUG [M:0;9f1eb259a29b:41689 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4cd7e0390398453296232501961ad367 as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/4cd7e0390398453296232501961ad367 2024-11-07T09:39:00,862 INFO [M:0;9f1eb259a29b:41689 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/4cd7e0390398453296232501961ad367, entries=8, sequenceid=2123, filesize=5.5 K 2024-11-07T09:39:00,863 DEBUG [M:0;9f1eb259a29b:41689 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/614df009066d46e29b8940e213b5ce7d as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/614df009066d46e29b8940e213b5ce7d 2024-11-07T09:39:00,865 INFO [M:0;9f1eb259a29b:41689 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 614df009066d46e29b8940e213b5ce7d 2024-11-07T09:39:00,865 INFO [M:0;9f1eb259a29b:41689 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/614df009066d46e29b8940e213b5ce7d, entries=166, sequenceid=2123, filesize=42.4 K 2024-11-07T09:39:00,866 DEBUG [M:0;9f1eb259a29b:41689 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/481e2383f1404caf8f82fd8bc5760c3d as hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/481e2383f1404caf8f82fd8bc5760c3d 2024-11-07T09:39:00,869 INFO [M:0;9f1eb259a29b:41689 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35443/user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/481e2383f1404caf8f82fd8bc5760c3d, entries=1, sequenceid=2123, filesize=5.0 K 2024-11-07T09:39:00,869 INFO [M:0;9f1eb259a29b:41689 {}] regionserver.HRegion(3140): Finished flush of dataSize ~750.27 KB/768279, heapSize ~921.56 KB/943680, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1278ms, sequenceid=2123, compaction requested=false 2024-11-07T09:39:00,871 INFO [M:0;9f1eb259a29b:41689 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T09:39:00,871 DEBUG [M:0;9f1eb259a29b:41689 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1730972339591Disabling compacts and flushes for region at 1730972339591Disabling writes for close at 1730972339591Obtaining lock to block concurrent updates at 1730972339591Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1730972339591Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=768279, getHeapSize=943920, getOffHeapSize=0, getCellsCount=2314 at 1730972339591Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1730972339592 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1730972339592Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1730972339606 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1730972339606Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1730972340013 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1730972340029 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1730972340029Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1730972340438 (+409 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1730972340452 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1730972340452Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ab9706c: reopening flushed file at 1730972340859 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@12dffa96: reopening flushed file at 1730972340862 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@64cef1ab: reopening flushed file at 1730972340866 (+4 ms)Finished flush of dataSize ~750.27 KB/768279, heapSize ~921.56 KB/943680, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1278ms, sequenceid=2123, compaction requested=false at 1730972340869 (+3 ms)Writing region close event to WAL at 1730972340871 (+2 ms)Closed at 1730972340871 2024-11-07T09:39:00,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34577 is added to blk_1073741830_1006 (size=907457) 2024-11-07T09:39:00,873 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/3086c4c1-6b60-e144-14f5-991cdbec3260/MasterData/WALs/9f1eb259a29b,41689,1730972153310/9f1eb259a29b%2C41689%2C1730972153310.1730972156021 not finished, retry = 0 2024-11-07T09:39:00,974 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-07T09:39:00,974 INFO [M:0;9f1eb259a29b:41689 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-07T09:39:00,975 INFO [M:0;9f1eb259a29b:41689 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41689 2024-11-07T09:39:00,975 INFO [M:0;9f1eb259a29b:41689 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-07T09:39:01,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41689-0x100f1c8dd440000, quorum=127.0.0.1:59189, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T09:39:01,077 INFO [M:0;9f1eb259a29b:41689 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-07T09:39:01,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41689-0x100f1c8dd440000, quorum=127.0.0.1:59189, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T09:39:01,082 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@292d50c4{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-07T09:39:01,084 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@667c6eba{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-07T09:39:01,085 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-07T09:39:01,085 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@72782cb0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-07T09:39:01,085 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@54a7b62e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5aa2b610-c3b5-4196-570b-200c5ef98b6f/hadoop.log.dir/,STOPPED} 2024-11-07T09:39:01,089 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-07T09:39:01,089 WARN [BP-1158119767-172.17.0.2-1730972150483 heartbeating to localhost/127.0.0.1:35443 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-07T09:39:01,089 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-07T09:39:01,089 WARN [BP-1158119767-172.17.0.2-1730972150483 heartbeating to localhost/127.0.0.1:35443 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1158119767-172.17.0.2-1730972150483 (Datanode Uuid 00b1f61d-c7c3-450c-b125-faf9dddf3851) service to localhost/127.0.0.1:35443 2024-11-07T09:39:01,091 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5aa2b610-c3b5-4196-570b-200c5ef98b6f/cluster_ccb59956-e5ab-933e-47b6-10a1e2a3fb72/data/data1/current/BP-1158119767-172.17.0.2-1730972150483 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-07T09:39:01,092 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5aa2b610-c3b5-4196-570b-200c5ef98b6f/cluster_ccb59956-e5ab-933e-47b6-10a1e2a3fb72/data/data2/current/BP-1158119767-172.17.0.2-1730972150483 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-07T09:39:01,092 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-07T09:39:01,102 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2341a969{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-07T09:39:01,103 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2d064751{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-07T09:39:01,103 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-07T09:39:01,103 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@130d168e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-07T09:39:01,103 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1a58fd8a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5aa2b610-c3b5-4196-570b-200c5ef98b6f/hadoop.log.dir/,STOPPED} 2024-11-07T09:39:01,124 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-07T09:39:01,261 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down